From 6bef2b3d95de3724757eb0ce21ac051ae133ad99 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Mon, 27 Apr 2026 12:08:24 +0300 Subject: [PATCH] Prepare mobile de parser release --- .dockerignore | 23 + .env.example | 45 + .gitattributes | 6 + .gitignore | 22 + Dockerfile | 32 + README.md | 3 + alembic.ini | 40 + alembic/env.py | 64 + alembic/script.py.mako | 25 + alembic/versions/001_initial.py | 103 + alembic/versions/002_add_indexes.py | 24 + alembic/versions/003_add_composite_indexes.py | 36 + alembic/versions/004_add_ingestion_tables.py | 17 + docker-compose.yml | 295 ++ entrypoint.sh | 89 + iaai_scraper/__init__.py | 1 + iaai_scraper/api/__init__.py | 3 + iaai_scraper/api/app.py | 40 + iaai_scraper/api/deps.py | 9 + iaai_scraper/api/routes/__init__.py | 0 iaai_scraper/api/routes/cars.py | 103 + iaai_scraper/api/routes/health.py | 30 + iaai_scraper/api/routes/tasks.py | 223 ++ iaai_scraper/browser/__init__.py | 6 + iaai_scraper/browser/factory.py | 214 ++ iaai_scraper/browser/fast_client.py | 863 +++++ iaai_scraper/browser/listing.py | 714 ++++ iaai_scraper/browser/network.py | 130 + iaai_scraper/browser/pace.py | 46 + iaai_scraper/cli.py | 168 + iaai_scraper/core/__init__.py | 1 + iaai_scraper/core/config.py | 434 +++ iaai_scraper/core/exceptions.py | 14 + iaai_scraper/core/logs.py | 55 + iaai_scraper/core/retry.py | 53 + iaai_scraper/core/runtime_config.py | 395 ++ iaai_scraper/core/utils.py | 72 + iaai_scraper/discovery/__init__.py | 15 + iaai_scraper/discovery/sitemap.py | 210 ++ iaai_scraper/fast_sync.py | 472 +++ iaai_scraper/mobile_de/__init__.py | 3 + iaai_scraper/mobile_de/client.py | 250 ++ iaai_scraper/mobile_de/flight.py | 79 + iaai_scraper/mobile_de/mapper.py | 220 ++ iaai_scraper/mobile_de/models.py | 35 + iaai_scraper/mobile_de/scraper.py | 325 ++ iaai_scraper/parsing/__init__.py | 1 + iaai_scraper/parsing/fast_mapper.py | 368 ++ iaai_scraper/parsing/mapper.py | 405 ++ iaai_scraper/parsing/parser.py | 408 ++ iaai_scraper/proxy_bridge.py | 317 ++ iaai_scraper/scraper.py | 2660 +++++++++++++ iaai_scraper/storage/__init__.py | 1 + iaai_scraper/storage/db.py | 676 ++++ iaai_scraper/storage/enums.py | 25 + iaai_scraper/storage/models.py | 82 + iaai_scraper/storage/schemas.py | 87 + iaai_scraper/worker/__init__.py | 3 + iaai_scraper/worker/celery_app.py | 207 + iaai_scraper/worker/self_heal.py | 276 ++ iaai_scraper/worker/tasks.py | 3358 +++++++++++++++++ pyproject.toml | 45 + runtime_config.json | 114 + tests/conftest.py | 7 + tests/test_db.py | 127 + tests/test_fast_client.py | 117 + tests/test_fast_sync.py | 141 + tests/test_listing.py | 68 + tests/test_mappers.py | 94 + tests/test_parser.py | 54 + tests/test_resilience.py | 79 + tests/test_scraper.py | 319 ++ tests/test_self_heal.py | 81 + tests/test_utils.py | 113 + tests/test_worker_tasks.py | 615 +++ uv.lock | 1000 +++++ 76 files changed, 17855 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/001_initial.py create mode 100644 alembic/versions/002_add_indexes.py create mode 100644 alembic/versions/003_add_composite_indexes.py create mode 100644 alembic/versions/004_add_ingestion_tables.py create mode 100644 docker-compose.yml create mode 100644 entrypoint.sh create mode 100644 iaai_scraper/__init__.py create mode 100644 iaai_scraper/api/__init__.py create mode 100644 iaai_scraper/api/app.py create mode 100644 iaai_scraper/api/deps.py create mode 100644 iaai_scraper/api/routes/__init__.py create mode 100644 iaai_scraper/api/routes/cars.py create mode 100644 iaai_scraper/api/routes/health.py create mode 100644 iaai_scraper/api/routes/tasks.py create mode 100644 iaai_scraper/browser/__init__.py create mode 100644 iaai_scraper/browser/factory.py create mode 100644 iaai_scraper/browser/fast_client.py create mode 100644 iaai_scraper/browser/listing.py create mode 100644 iaai_scraper/browser/network.py create mode 100644 iaai_scraper/browser/pace.py create mode 100644 iaai_scraper/cli.py create mode 100644 iaai_scraper/core/__init__.py create mode 100644 iaai_scraper/core/config.py create mode 100644 iaai_scraper/core/exceptions.py create mode 100644 iaai_scraper/core/logs.py create mode 100644 iaai_scraper/core/retry.py create mode 100644 iaai_scraper/core/runtime_config.py create mode 100644 iaai_scraper/core/utils.py create mode 100644 iaai_scraper/discovery/__init__.py create mode 100644 iaai_scraper/discovery/sitemap.py create mode 100644 iaai_scraper/fast_sync.py create mode 100644 iaai_scraper/mobile_de/__init__.py create mode 100644 iaai_scraper/mobile_de/client.py create mode 100644 iaai_scraper/mobile_de/flight.py create mode 100644 iaai_scraper/mobile_de/mapper.py create mode 100644 iaai_scraper/mobile_de/models.py create mode 100644 iaai_scraper/mobile_de/scraper.py create mode 100644 iaai_scraper/parsing/__init__.py create mode 100644 iaai_scraper/parsing/fast_mapper.py create mode 100644 iaai_scraper/parsing/mapper.py create mode 100644 iaai_scraper/parsing/parser.py create mode 100644 iaai_scraper/proxy_bridge.py create mode 100644 iaai_scraper/scraper.py create mode 100644 iaai_scraper/storage/__init__.py create mode 100644 iaai_scraper/storage/db.py create mode 100644 iaai_scraper/storage/enums.py create mode 100644 iaai_scraper/storage/models.py create mode 100644 iaai_scraper/storage/schemas.py create mode 100644 iaai_scraper/worker/__init__.py create mode 100644 iaai_scraper/worker/celery_app.py create mode 100644 iaai_scraper/worker/self_heal.py create mode 100644 iaai_scraper/worker/tasks.py create mode 100644 pyproject.toml create mode 100644 runtime_config.json create mode 100644 tests/conftest.py create mode 100644 tests/test_db.py create mode 100644 tests/test_fast_client.py create mode 100644 tests/test_fast_sync.py create mode 100644 tests/test_listing.py create mode 100644 tests/test_mappers.py create mode 100644 tests/test_parser.py create mode 100644 tests/test_resilience.py create mode 100644 tests/test_scraper.py create mode 100644 tests/test_self_heal.py create mode 100644 tests/test_utils.py create mode 100644 tests/test_worker_tasks.py create mode 100644 uv.lock diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6079228 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +__pycache__/ +.pytest_cache/ +.venv/ +venv/ + +.coverage +coverage.xml + +*.pyc +*.pyo +*.pyd +*.log +*.db + +.env +.git/ +.vscode/ + +*.egg-info/ +dist/ +build/ +artifacts/ +tokens_data/ \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..958f573 --- /dev/null +++ b/.env.example @@ -0,0 +1,45 @@ +# mobile.de scraper Docker defaults + +# PostgreSQL used by docker-compose. +POSTGRES_USER=mobilede +POSTGRES_PASSWORD=mobilede +POSTGRES_DB=mobilede_scraper +MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper + +# Legacy env names still consumed by Settings until the old IAAI core is fully removed. +IAAI_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper +IAAI_DATABASE_ECHO=false +IAAI_DATABASE_POOL_SIZE=5 +IAAI_DATABASE_MAX_OVERFLOW=5 +IAAI_DATABASE_POOL_RECYCLE_SECONDS=1800 + +# Redis / Celery stack. +IAAI_REDIS_URL=redis://redis:6379/0 +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 +CELERY_TASK_SOFT_TIME_LIMIT=86400 +CELERY_TASK_TIME_LIMIT=86520 +CELERY_BROKER_VISIBILITY_TIMEOUT=90000 +CELERY_WORKER_CONCURRENCY=1 +CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 +IAAI_STARTUP_SYNC_ENABLED=true +MOBILEDE_STARTUP_MAX_PAGES=5 +MOBILEDE_BEAT_MAX_PAGES=5 + +# mobile.de one-shot CLI services. +MOBILEDE_SEARCH_START_PAGE=1 +MOBILEDE_SEARCH_MAX_PAGES=1 +MOBILEDE_REQUEST_DELAY_SECONDS=0.7 +MOBILEDE_SYNC_LANE=mobile_de_cars +MOBILEDE_LISTING_ID=449929602 + +# Logging / runtime. +IAAI_LOG_LEVEL=INFO +IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json +TZ=UTC + +# Legacy browser settings kept for old deprecated IAAI commands only. +IAAI_HEADLESS=true +IAAI_BROWSER_ENGINE=chromium +IAAI_TOKENS_FILE=/data/tokens.json +IAAI_SELF_HEAL_ENABLED=true diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b06c566 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Force LF for shell scripts (critical for Docker on Windows) +*.sh text eol=lf +entrypoint.sh text eol=lf + +# Default for other files +* text=auto diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0c27e35 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +__pycache__/ +.pytest_cache/ +.coverage +coverage.xml +.env* +!.env.example +.venv/ +venv/ +*.pyc +*.pyo +*.pyd +*.log +*.db +storage_state.json +.vscode/ +*.egg-info/ +dist/ +build/ +artifacts/ +celerybeat-schedule* +tokens_data/ +tokens.json \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b35903e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM mcr.microsoft.com/playwright/python:v1.58.0-noble + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN groupadd --system app && useradd --system --gid app --create-home app + +WORKDIR /app + +COPY pyproject.toml uv.lock ./ +RUN pip install --no-cache-dir uv \ + && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project -o /tmp/requirements.txt \ + && pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir playwright-stealth + +# Ставим Chromium и Firefox. +# По умолчанию используем Chromium. +RUN python -m playwright install chromium firefox + +COPY . . +RUN pip install --no-cache-dir -e . +RUN python -m compileall -q iaai_scraper +RUN chmod +x entrypoint.sh +RUN mkdir -p /data && chown -R app:app /app /data + +STOPSIGNAL SIGINT + +USER app + +# По умолчанию запускаем API. +ENTRYPOINT ["./entrypoint.sh"] +CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..6a82af4 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# mobile.de Scraper + +Парсер [`mobile.de`](https://www.mobile.de/ru). diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..f4be675 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +# Alembic Configuration File + +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..9f9ae88 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,64 @@ +# Alembic env.py — подключение к БД через Settings. + +import os +import sys +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +# Добавляем корень проекта в sys.path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from iaai_scraper.core.config import Settings +from iaai_scraper.storage.models import Base + +config = context.config +VERSION_TABLE = "iaai_parser_alembic_version" + +# Logging +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Подставляем URL из Settings (env vars имеют приоритет) +settings = Settings() +config.set_main_option("sqlalchemy.url", settings.database.url) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + # Run migrations in 'offline' mode. + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + version_table=VERSION_TABLE, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + # Run migrations in 'online' mode. + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table=VERSION_TABLE, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..958df87 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py new file mode 100644 index 0000000..454207b --- /dev/null +++ b/alembic/versions/001_initial.py @@ -0,0 +1,103 @@ +# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _schema_name() -> str | None: + bind = op.get_bind() + return "public" if bind.dialect.name == "postgresql" else None + + +def _table_exists(table_name: str) -> bool: + inspector = sa.inspect(op.get_bind()) + return table_name in inspector.get_table_names(schema=_schema_name()) + + +def _index_exists(table_name: str, index_name: str) -> bool: + if not _table_exists(table_name): + return False + inspector = sa.inspect(op.get_bind()) + indexes = inspector.get_indexes(table_name, schema=_schema_name()) + return any(index.get("name") == index_name for index in indexes) + + +def upgrade() -> None: + # Таблица iaai_cars — аукционные машины с IAAI + if not _table_exists("iaai_cars"): + op.create_table( + "iaai_cars", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("parser_id", sa.String(50), nullable=False, unique=True), + sa.Column("brand", sa.String(50), nullable=False), + sa.Column("model", sa.String(50), nullable=False), + sa.Column("year", sa.Integer, nullable=True), + sa.Column("price", sa.BigInteger, nullable=True), + sa.Column("currency", sa.String(10), nullable=False, server_default="USD"), + sa.Column("mileage", sa.Integer, nullable=False, server_default="0"), + sa.Column("country", sa.String(10), nullable=False, server_default="NA"), + sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("color", sa.String, nullable=False, server_default="other"), + sa.Column("drive", sa.String(10), nullable=True), + sa.Column("gearbox", sa.String(10), nullable=True), + sa.Column("steering_wheel", sa.String(10), nullable=True), + sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"), + sa.Column("engine_volume", sa.Integer, nullable=True), + sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"), + sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("origin", sa.String(20), nullable=False, server_default="NA"), + sa.Column("origin_url", sa.String, nullable=False), + sa.Column("origin_id", sa.String, nullable=False, unique=True), + sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("evaluation", sa.String, nullable=True), + sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")), + sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("slug", sa.String, nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + + if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"): + op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"]) + if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"): + op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True) + + # Таблица iaai_images — изображения машин + if not _table_exists("iaai_images"): + op.create_table( + "iaai_images", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("fullres_image", sa.String, nullable=False), + sa.Column("preview_image", sa.String, nullable=False), + sa.Column("order_index", sa.Integer, nullable=False), + sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False), + ) + + # Таблица iaai_sync_runs — логирование синхронизаций + if not _table_exists("iaai_sync_runs"): + op.create_table( + "iaai_sync_runs", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("status", sa.Text, nullable=False), + sa.Column("lane", sa.Text, nullable=False), + sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"), + sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("error_summary", sa.Text, nullable=True), + ) + + +def downgrade() -> None: + # Compatibility-only migration for shared production databases. + pass diff --git a/alembic/versions/002_add_indexes.py b/alembic/versions/002_add_indexes.py new file mode 100644 index 0000000..6fb5ad5 --- /dev/null +++ b/alembic/versions/002_add_indexes.py @@ -0,0 +1,24 @@ +# Индексы производительности +from typing import Sequence, Union + +from alembic import op + +revision: str = "002_add_indexes" +down_revision: Union[str, None] = "001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)") + op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)") + op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)") + op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)") + op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)") + op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)") + op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)") + + +def downgrade() -> None: + # Compatibility-only migration for shared production databases. + pass diff --git a/alembic/versions/003_add_composite_indexes.py b/alembic/versions/003_add_composite_indexes.py new file mode 100644 index 0000000..393f08c --- /dev/null +++ b/alembic/versions/003_add_composite_indexes.py @@ -0,0 +1,36 @@ +# Индексы для масштабированной БД (20k+ записей) +from typing import Sequence, Union + +from alembic import op + +revision: str = "003_add_composite_indexes" +down_revision: Union[str, None] = "002_add_indexes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Partial index для активных машин IAAI (is_sold = FALSE) + # Ускоряет поиск при mark_sold операциях + op.execute( + "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai " + "ON iaai_cars (origin_url) " + "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'" + ) + + # Composite index для проверки дубликатов изображений + op.execute( + "CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres " + "ON iaai_images (car_id, fullres_image)" + ) + + # Index для быстрого поиска existing машин по origin_url + op.execute( + "CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id " + "ON iaai_cars (origin_url, origin_id)" + ) + + +def downgrade() -> None: + # Compatibility-only migration for shared production databases. + pass diff --git a/alembic/versions/004_add_ingestion_tables.py b/alembic/versions/004_add_ingestion_tables.py new file mode 100644 index 0000000..3ce4466 --- /dev/null +++ b/alembic/versions/004_add_ingestion_tables.py @@ -0,0 +1,17 @@ +# Placeholder migration (ingestion tables not yet needed) +from typing import Sequence, Union + +from alembic import op + +revision: str = "004_add_ingestion_tables" +down_revision: Union[str, None] = "003_add_composite_indexes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..c9d9f10 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,295 @@ +x-app-env: &app-env + # Legacy env name is still used by Settings; values are mobile.de defaults. + IAAI_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper} + MOBILEDE_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper} + IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} + CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} + IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800} + IAAI_DATABASE_POOL_SIZE: ${IAAI_DATABASE_POOL_SIZE:-20} + IAAI_DATABASE_MAX_OVERFLOW: ${IAAI_DATABASE_MAX_OVERFLOW:-40} + CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} + CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} + CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} + IAAI_PROFILE: ${IAAI_PROFILE:-fast} + IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}} + IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true} + IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false} + IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false} + IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1} + IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-2} + IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0} + IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1} + IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1} + # 0 = без лимита. + CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} + CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4} + CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork} + CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} + CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000} + MOBILEDE_STARTUP_MAX_PAGES: ${MOBILEDE_STARTUP_MAX_PAGES:-100} + MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100} + MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0} + MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2} + MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true} + MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true} + MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-0} + MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-100} + MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true} + MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true} + MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2} + MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10} + MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1800} + MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true} + MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false} + MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true} + MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true} + MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true} + MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1} + IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true} + IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0} + IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9} + IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8} + IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-32} + IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true} + IAAI_FILTERED_SEARCH_URL: ${IAAI_FILTERED_SEARCH_URL:-} + IAAI_FILTERED_SEARCH_URLS: ${IAAI_FILTERED_SEARCH_URLS:-} + IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-runtime} + IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999} + IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000} + IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true} + IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true} + IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json} + IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} + IAAI_BROWSER_ENGINE: chromium + # Self-heal для worker. + IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true} + IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} + IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900} + IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} + IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} + # Если РІ Postgres нет записей дольше 60 РјРёРЅСѓС‚ РїСЂРё активной работе — полный restart СЃ segment 1. + IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600} + TZ: ${TZ:-UTC} + +x-env-file: &env-file + - path: .env + required: false + +x-app-service: &app-service + build: . + env_file: *env-file + environment: *app-env + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + +x-worker-service: &worker-service + <<: *app-service + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + - tokens_data:/data + +services: + # PostgreSQL. + postgres: + image: postgres:16-alpine + container_name: mobilede-postgres + restart: unless-stopped + environment: + POSTGRES_USER: ${POSTGRES_USER:-mobilede} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mobilede} + POSTGRES_DB: ${POSTGRES_DB:-mobilede_scraper} + ports: + - "127.0.0.1:5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mobilede} -d ${POSTGRES_DB:-mobilede_scraper}"] + interval: 5s + timeout: 3s + retries: 5 + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Redis. + redis: + image: redis:7-alpine + container_name: mobilede-redis + restart: unless-stopped + ports: + - "127.0.0.1:6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + command: > + redis-server + --appendonly yes + --save 60 1000 + volumes: + - redisdata:/data + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Миграции. + migrate: + <<: *app-service + container_name: mobilede-migrate + restart: "no" + depends_on: + postgres: + condition: service_healthy + command: alembic upgrade head + + # API. + api: + <<: *app-service + container_name: mobilede-api + restart: unless-stopped + ports: + - "127.0.0.1:8000:8000" + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + command: > + uvicorn iaai_scraper.api.app:app + --host 0.0.0.0 --port 8000 --workers 1 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + stop_grace_period: 30s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Worker. + worker: + <<: *worker-service + container_name: mobilede-worker + restart: unless-stopped + init: true + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + stop_grace_period: 60s + command: > + celery -A iaai_scraper.worker.celery_app worker + --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork} + --pidfile=/tmp/celery-worker.pid + -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} + healthcheck: + # Проверка worker. + test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 90s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # Beat. + beat: + <<: *worker-service + container_name: mobilede-beat + restart: unless-stopped + init: true + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + command: > + celery -A iaai_scraper.worker.celery_app beat + --loglevel=info + --pidfile=/tmp/celery-beat.pid + --schedule=/tmp/celerybeat-schedule + healthcheck: + test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 90s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + + # One-shot CLI: collect mobile.de search pages into artifacts/json. + mobilede-search: + <<: *app-service + container_name: mobilede-search + profiles: ["cli"] + restart: "no" + depends_on: + migrate: + condition: service_completed_successfully + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + - ./artifacts:/app/artifacts + command: > + mobilede search + --max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1} + --page ${MOBILEDE_SEARCH_START_PAGE:-1} + --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7} + --output /app/artifacts/json/mobilede_search.json + + # One-shot CLI: collect and upsert mobile.de search pages into Postgres. + mobilede-sync-search: + <<: *app-service + container_name: mobilede-sync-search + profiles: ["cli"] + restart: "no" + depends_on: + migrate: + condition: service_completed_successfully + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + - ./artifacts:/app/artifacts + command: > + mobilede sync-search + --max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1} + --page ${MOBILEDE_SEARCH_START_PAGE:-1} + --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7} + --lane ${MOBILEDE_SYNC_LANE:-mobile_de_cars} + --output /app/artifacts/json/mobilede_sync_search.json + + # One-shot CLI: collect one detail page by listing id. + mobilede-detail: + <<: *app-service + container_name: mobilede-detail + profiles: ["cli"] + restart: "no" + depends_on: + migrate: + condition: service_completed_successfully + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + - ./artifacts:/app/artifacts + command: > + mobilede detail ${MOBILEDE_LISTING_ID:-449929602} + --output /app/artifacts/json/mobilede_detail.json + +volumes: + pgdata: + redisdata: + tokens_data: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..53a9fd0 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,89 @@ +#!/bin/bash +set -euo pipefail + +is_worker_command() { + local joined="$*" + case "$joined" in + *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*) + return 0 + ;; + esac + return 1 +} + +is_scrape_cli_command() { + local joined="$*" + case "$joined" in + *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*) + return 0 + ;; + esac + return 1 +} + +needs_browser_runtime() { + is_worker_command "$@" || is_scrape_cli_command "$@" +} + +start_proxy_bridge_if_needed() { + if ! needs_browser_runtime "$@"; then + return 0 + fi + + if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then + return 0 + fi + + echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..." + + if [ -z "${IAAI_PROXY_SERVER:-}" ]; then + export IAAI_PROXY_SERVER="http://127.0.0.1:8899" + elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then + export IAAI_PROXY_SERVER="http://127.0.0.1:8899" + fi + + echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}" + python -m iaai_scraper.proxy_bridge & + BRIDGE_PID=$! + sleep 1 + + if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then + echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start" + exit 1 + fi + + echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})" +} + +start_self_heal_watchdog_if_worker() { + if ! is_worker_command "$@"; then + return 0 + fi + + # После reboot/restart контейнера файловая система контейнера сохраняется, + # поэтому stale pidfile может остаться от прошлого запуска и блокировать старт Celery. + # Удаляем его перед запуском worker-процесса. + rm -f /tmp/celery-worker.pid + + if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then + echo "[entrypoint] Self-heal watchdog disabled" + return 0 + fi + + echo "[entrypoint] Starting self-heal watchdog for worker..." + python -m iaai_scraper.worker.self_heal & + SELF_HEAL_PID=$! + sleep 1 + + if ! kill -0 "${SELF_HEAL_PID}" 2>/dev/null; then + echo "[entrypoint] ERROR: self-heal watchdog failed to start" + exit 1 + fi + + echo "[entrypoint] Self-heal watchdog started (PID ${SELF_HEAL_PID})" +} + +start_proxy_bridge_if_needed "$@" +start_self_heal_watchdog_if_worker "$@" + +exec "$@" diff --git a/iaai_scraper/__init__.py b/iaai_scraper/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/api/__init__.py b/iaai_scraper/api/__init__.py new file mode 100644 index 0000000..b94a1e8 --- /dev/null +++ b/iaai_scraper/api/__init__.py @@ -0,0 +1,3 @@ +from .app import create_app + +__all__ = ["create_app"] diff --git a/iaai_scraper/api/app.py b/iaai_scraper/api/app.py new file mode 100644 index 0000000..ac2215c --- /dev/null +++ b/iaai_scraper/api/app.py @@ -0,0 +1,40 @@ +# Создание FastAPI-приложения и настройка его жизненного цикла. + +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from ..core.config import Settings +from ..storage.db import PersistenceService +from .routes import cars, health, tasks + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Жизненный цикл. + # Таблицы создаются через Alembic-миграции (сервис migrate). + yield + + +def create_app(settings: Settings | None = None) -> FastAPI: + _settings = settings or Settings() + + app = FastAPI( + title="mobile.de Scraper API", + description="REST API для управления задачами скрапинга mobile.de и просмотра данных", + version="1.0.0", + lifespan=lifespan, + ) + + app.state.settings = _settings + app.state.persistence = PersistenceService(_settings) + + app.include_router(health.router, tags=["health"]) + app.include_router(cars.router, prefix="/api/v1", tags=["cars"]) + app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"]) + + return app + + +# Экземпляр приложения для запуска через uvicorn. +app = create_app() diff --git a/iaai_scraper/api/deps.py b/iaai_scraper/api/deps.py new file mode 100644 index 0000000..86b4bc3 --- /dev/null +++ b/iaai_scraper/api/deps.py @@ -0,0 +1,9 @@ +# Вспомогательные зависимости для FastAPI-роутов. + +from fastapi import Request + +from ..storage.db import PersistenceService + + +def get_persistence(request: Request) -> PersistenceService: + return request.app.state.persistence diff --git a/iaai_scraper/api/routes/__init__.py b/iaai_scraper/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/iaai_scraper/api/routes/cars.py b/iaai_scraper/api/routes/cars.py new file mode 100644 index 0000000..7195d48 --- /dev/null +++ b/iaai_scraper/api/routes/cars.py @@ -0,0 +1,103 @@ +# Роуты для просмотра автомобилей и агрегированной статистики. + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import Car, Image +from ...storage.schemas import CarRead + +router = APIRouter() + + +@router.get("/cars") +def list_cars( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + brand: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + is_sold: bool | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + # Список автомобилей с пагинацией и фильтрами + with persistence.session_scope() as session: + query = select(Car) + + if brand: + query = query.where(Car.brand.ilike(f"%{brand}%")) + if model: + query = query.where(Car.model.ilike(f"%{model}%")) + if year_min is not None: + query = query.where(Car.year >= year_min) + if year_max is not None: + query = query.where(Car.year <= year_max) + if is_sold is not None: + query = query.where(Car.is_sold == is_sold) + + count_query = select(func.count()).select_from(query.subquery()) + total = session.execute(count_query).scalar() or 0 + + offset = (page - 1) * per_page + cars = session.execute( + query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if per_page else 0, + "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars], + } + + +@router.get("/cars/{car_id}") +def get_car( + car_id: int, + persistence: PersistenceService = Depends(get_persistence), +): + # Детальная информация об автомобиле с изображениями + with persistence.session_scope() as session: + car = session.get(Car, car_id) + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/cars/by-origin/{origin_id}") +def get_car_by_origin( + origin_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + # Поиск автомобиля по origin_id + with persistence.session_scope() as session: + car = session.execute( + select(Car).where(Car.origin_id == origin_id) + ).scalars().first() + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/stats") +def get_stats(persistence: PersistenceService = Depends(get_persistence)): + # Общая статистика по БД + with persistence.session_scope() as session: + total_cars = session.execute(select(func.count(Car.id))).scalar() or 0 + total_images = session.execute(select(func.count(Image.id))).scalar() or 0 + brands = session.execute( + select(Car.brand, func.count(Car.id)) + .group_by(Car.brand) + .order_by(func.count(Car.id).desc()) + .limit(20) + ).all() + + return { + "total_cars": total_cars, + "total_images": total_images, + "top_brands": [{"brand": b, "count": c} for b, c in brands], + } + diff --git a/iaai_scraper/api/routes/health.py b/iaai_scraper/api/routes/health.py new file mode 100644 index 0000000..78b96f3 --- /dev/null +++ b/iaai_scraper/api/routes/health.py @@ -0,0 +1,30 @@ +# Роут проверки доступности сервиса и соединения с БД. + +import logging + +from fastapi import APIRouter, Depends +from sqlalchemy import text + +from ..deps import get_persistence +from ...storage.db import PersistenceService + +router = APIRouter() +logger = logging.getLogger("iaai_scraper.api.health") + + +@router.get("/health") +def health_check(persistence: PersistenceService = Depends(get_persistence)): + # Проверка API и БД + db_ok = False + try: + with persistence.session_scope() as session: + session.execute(text("SELECT 1")) + db_ok = True + except Exception: + logger.warning("Health DB check failed", exc_info=True) + + return { + "status": "ok" if db_ok else "degraded", + "service": "mobilede-scraper-api", + "database": "connected" if db_ok else "unavailable", + } diff --git a/iaai_scraper/api/routes/tasks.py b/iaai_scraper/api/routes/tasks.py new file mode 100644 index 0000000..4e1e2a1 --- /dev/null +++ b/iaai_scraper/api/routes/tasks.py @@ -0,0 +1,223 @@ +# Роуты запуска задач синхронизации и просмотра истории sync-runs. + +import json + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from redis import Redis +from sqlalchemy import select, func + +from ...core.config import Settings +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import SyncRun +from ...worker.celery_app import IAAI_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app +from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task + +router = APIRouter() + + +class SyncVehicleRequest(BaseModel): + vehicle_url: str + lane: str = "iaai" + + +class SyncListingRequest(BaseModel): + make: str | None = None + model: str | None = None + lane: str = "iaai_cars" + limit: int | None = None + only_new: bool | None = None + + +class MobileDeSyncSearchRequest(BaseModel): + start_page: int = 1 + max_pages: int = 5 + lane: str = "mobile_de_cars" + search_url: str | None = None + make_id: str | None = None + model_id: str | None = None + price_min: str | None = None + price_max: str | None = None + year_min: str | None = None + year_max: str | None = None + delay_seconds: float = 0.7 + use_cursor: bool = False + continuous: bool = True + + +class MobileDeSyncDetailRequest(BaseModel): + listing_id: str + lane: str = "mobile_de_cars" + + +class MobileDeRuntimeSegmentsRequest(BaseModel): + lane: str = "mobile_de_cars" + delay_seconds: float = 0.7 + use_cursor: bool = True + continuous: bool = True + + +@router.post("/mobilede/tasks/sync-search") +def start_mobilede_sync_search(body: MobileDeSyncSearchRequest): + result = mobilede_sync_search_task.apply_async( + kwargs=body.model_dump(), + queue=MOBILEDE_SYNC_QUEUE, + ) + return { + "task_id": result.id, + "status": "queued", + "queue": MOBILEDE_SYNC_QUEUE, + } + + +@router.post("/mobilede/tasks/sync-detail") +def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest): + result = mobilede_sync_detail_task.apply_async( + kwargs=body.model_dump(), + queue=MOBILEDE_SYNC_QUEUE, + ) + return { + "task_id": result.id, + "status": "queued", + "queue": MOBILEDE_SYNC_QUEUE, + "listing_id": body.listing_id, + } + + +@router.post("/mobilede/tasks/sync-runtime-segments") +def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest): + result = mobilede_sync_runtime_segments_task.apply_async( + kwargs=body.model_dump(), + queue=MOBILEDE_SYNC_QUEUE, + ) + return { + "task_id": result.id, + "status": "queued", + "queue": MOBILEDE_SYNC_QUEUE, + } + + +@router.post("/tasks/sync-vehicle") +def start_sync_vehicle( + body: SyncVehicleRequest, +): + # Запустить задачу скрапинга одного автомобиля через Celery + result = sync_vehicle_task.apply_async( + kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, + queue=IAAI_SYNC_QUEUE, + ) + + return { + "task_id": result.id, + "status": "queued", + "vehicle_url": body.vehicle_url, + } + + +@router.post("/tasks/sync-listing") +def start_sync_listing( + body: SyncListingRequest, +): + # Запустить задачу полного цикла листинга через Celery + result = sync_listing_task.apply_async( + kwargs={ + "make": body.make, + "model": body.model, + "lane": body.lane, + "limit": body.limit, + "only_new": body.only_new, + }, + queue=IAAI_SYNC_QUEUE, + ) + + return { + "task_id": result.id, + "status": "queued", + } + + +@router.get("/tasks/{task_id}") +def get_task_status(task_id: str): + result = celery_app.AsyncResult(task_id) + + payload: dict = { + "task_id": task_id, + "state": result.state, + } + + if result.successful(): + payload["result"] = result.result + elif result.failed(): + payload["error"] = str(result.result) + elif result.info is not None: + payload["meta"] = result.info + + progress = _read_task_progress(task_id) + if progress is not None: + payload["progress"] = progress + + return payload + + +def _read_task_progress(task_id: str) -> dict | None: + redis_client = None + try: + settings = Settings() + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + raw = redis_client.get(f"iaai:state:task_progress:{task_id}") + if not raw: + return None + data = json.loads(raw) + return data if isinstance(data, dict) else None + except Exception: + return None + finally: + if redis_client is not None: + try: + redis_client.close() + except Exception: + pass + + +@router.get("/sync-runs") +def list_sync_runs( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + persistence: PersistenceService = Depends(get_persistence), +): + # История запусков синхронизации + with persistence.session_scope() as session: + total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 + offset = (page - 1) * per_page + runs = session.execute( + select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": r.id, + "started_at": r.started_at.isoformat() if r.started_at else None, + "finished_at": r.finished_at.isoformat() if r.finished_at else None, + "status": r.status, + "lane": r.lane, + "ids_fetched": r.ids_fetched, + "cars_upserted": r.cars_upserted, + "cars_failed": r.cars_failed, + "images_upserted": r.images_upserted, + "error_summary": r.error_summary, + } + for r in runs + ], + } diff --git a/iaai_scraper/browser/__init__.py b/iaai_scraper/browser/__init__.py new file mode 100644 index 0000000..38bb3f4 --- /dev/null +++ b/iaai_scraper/browser/__init__.py @@ -0,0 +1,6 @@ +from .factory import BrowserFactory +from .listing import ListingCollector +from .network import NetworkCapture +from .pace import HumanPacer + +__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"] diff --git a/iaai_scraper/browser/factory.py b/iaai_scraper/browser/factory.py new file mode 100644 index 0000000..ccba6cf --- /dev/null +++ b/iaai_scraper/browser/factory.py @@ -0,0 +1,214 @@ +import json +import logging +import random + +from playwright.sync_api import Browser, BrowserContext, Playwright + +try: + from playwright_stealth import stealth_sync +except ImportError: + stealth_sync = None + +from ..core.config import Settings + +logger = logging.getLogger("iaai_scraper.browser") + + +def _build_init_script() -> str: + # Маскировка браузера. + hardware_concurrency = random.choice([4, 8, 12, 16]) + device_memory = random.choice([4, 8, 16]) + languages = ["en-US", "en"] + + return f""" +(() => {{ + const define = (obj, prop, value) => {{ + try {{ + Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); + }} catch (e) {{}} + }}; + + define(navigator, 'webdriver', undefined); + define(navigator, 'platform', 'Win32'); + define(navigator, 'vendor', 'Google Inc.'); + define(navigator, 'language', '{languages[0]}'); + define(navigator, 'languages', {json.dumps(languages)}); + define(navigator, 'hardwareConcurrency', {hardware_concurrency}); + define(navigator, 'deviceMemory', {device_memory}); + define(navigator, 'maxTouchPoints', 0); + + if (!window.chrome) {{ + Object.defineProperty(window, 'chrome', {{ + value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, + configurable: true + }}); + }} + + const originalQuery = navigator.permissions && navigator.permissions.query; + if (originalQuery) {{ + navigator.permissions.query = (params) => ( + params && params.name === 'notifications' + ? Promise.resolve({{ state: Notification.permission }}) + : originalQuery(params) + ); + }} + + const originalGetParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(parameter) {{ + if (parameter === 37445) return 'Intel Inc.'; + if (parameter === 37446) return 'Intel Iris OpenGL Engine'; + return originalGetParameter.call(this, parameter); + }}; +}})(); +""" + + +class BrowserFactory: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def _resolve_engine(self) -> str: + # Выбор движка. + engine = self.settings.browser_engine.strip().lower() + if engine == "auto": + # Для IAAI стабильнее Chromium. + return "chromium" + if engine in ("firefox", "chromium"): + return engine + logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine) + return "chromium" + + def create_browser(self, playwright: Playwright) -> Browser: + engine = self._resolve_engine() + proxy_dict = self.settings.proxy.to_playwright_dict() + logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine) + + if engine == "firefox": + launch_kwargs: dict = {"headless": self.settings.headless} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + # Настройки Firefox. + launch_kwargs["firefox_user_prefs"] = { + "dom.webdriver.enabled": False, + "useAutomationExtension": False, + # Базовые оптимизации. + "media.autoplay.default": 5, + "media.volume_scale": "0.0", + "media.audio.playback.standalone": False, + "dom.ipc.processCount": 1, + "dom.ipc.plugins.enabled": False, + "browser.cache.disk.enable": False, + "browser.cache.memory.enable": True, + "browser.cache.memory.max_entry_size": 8192, + "network.prefetch-next": False, + "network.dns.disablePrefetch": True, + "permissions.default.image": 2, + "javascript.options.mem.gc_incremental_mark_slice_ms": 20, + } + logger.info("Launching Firefox (headless=%s)", self.settings.headless) + return playwright.firefox.launch(**launch_kwargs) + + # Запуск Chromium. + args = [ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-features=IsolateOrigins,site-per-process", + ] + if self.settings.headless: + args.append("--headless=new") + pw_headless = False + logger.info("Using Chromium new-headless mode (--headless=new)") + else: + pw_headless = False + + launch_kwargs = {"headless": pw_headless, "args": args} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + try: + logger.info("Trying to launch real Chrome channel") + return playwright.chromium.launch(channel="chrome", **launch_kwargs) + except Exception: + logger.warning("Chrome channel launch failed, falling back to Chromium") + return playwright.chromium.launch(**launch_kwargs) + + def create_context(self, browser: Browser) -> BrowserContext: + viewport = random.choice(self.settings.fingerprint.viewport_presets) + timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) + color_scheme = random.choice(["light", "dark"]) + + is_firefox = browser.browser_type.name == "firefox" + + ctx_kwargs: dict = { + "viewport": viewport, + "screen": viewport, + "locale": self.settings.fingerprint.locale, + "timezone_id": timezone_id, + "color_scheme": color_scheme, + "java_script_enabled": True, + "ignore_https_errors": False, + } + + if is_firefox: + # Заголовки Firefox. + ctx_kwargs["user_agent"] = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + } + else: + ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent + ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25]) + ctx_kwargs["is_mobile"] = False + ctx_kwargs["has_touch"] = False + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', + } + + context = browser.new_context(**ctx_kwargs) + context.set_default_timeout(self.settings.default_timeout_ms) + context.set_default_navigation_timeout(self.settings.default_timeout_ms) + + if not is_firefox: + # Маскировка Chromium. + context.add_init_script(_build_init_script()) + if stealth_sync: + context.on("page", lambda page: stealth_sync(page)) + logger.debug("playwright-stealth attached to context") + + return context + + @staticmethod + def enable_resource_blocking(page) -> None: + # Блокируем тяжёлые ресурсы. + BLOCKED_TYPES = {"image", "stylesheet", "font", "media"} + BLOCKED_URL_PATTERNS = ( + "google-analytics", "googletagmanager", "facebook.net", + "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2", + "analytics", "tracking", "adservice", + ) + + def _handle_route(route): + req = route.request + if req.resource_type in BLOCKED_TYPES: + route.abort() + return + url = req.url.lower() + if any(pat in url for pat in BLOCKED_URL_PATTERNS): + route.abort() + return + route.continue_() + + page.route("**/*", _handle_route) diff --git a/iaai_scraper/browser/fast_client.py b/iaai_scraper/browser/fast_client.py new file mode 100644 index 0000000..205857e --- /dev/null +++ b/iaai_scraper/browser/fast_client.py @@ -0,0 +1,863 @@ +from __future__ import annotations + +import html +import json +import logging +import math +import re +import threading +import time +from dataclasses import dataclass +from typing import Any, Iterator +from urllib.parse import quote, urljoin + +import requests +from requests.adapters import HTTPAdapter + +from ..core.config import Settings + +logger = logging.getLogger("iaai_scraper.fast_client") + +TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504} +CHALLENGE_MARKERS = ( + "_incapsula_resource", + "incapsula", + "incident id", + "request unsuccessful", + "access denied", +) +COOKIE_ACCEPT_SELECTORS = ( + "button:has-text('Accept All')", + "button:has-text('Accept all')", + "button:has-text('I Agree')", + "button:has-text('Agree')", + "button:has-text('Only necessary')", + "button:has-text('Только необходимые')", + "button:has-text('Принять все')", + "[id*='accept']", + "[class*='accept']", +) +LISTING_MARKER = 'id="GBPSearchQuery"' +DETAIL_MARKER = 'id="ProductDetailsVM"' +RESIZER_URL = "https://vis.iaai.com/resizer" +BRAND_SCOPE_OVERRIDES = { + # IAAI does not resolve every rare make through /Vehiclelisting/Cars/{make}. + # CUPRA is available through a saved Search scope URL from the site UI. + "CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d", +} +DEFAULT_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" +) +PLAYWRIGHT_REFRESH_POLLS = 8 + + +@dataclass(frozen=True) +class FastListingVehicle: + inventory_id: str + tenant: str | None + auction_id: str | None + auction_date: str | None + inventory_status: str | None + currency: str | None + timed_auction_closed: bool + timed_auction_indicator: bool + prebid_indicator: bool + buynow_indicator: bool + + +@dataclass(frozen=True) +class FastListingPage: + vehicles: list[FastListingVehicle] + result_count: int + page_size: int + current_page: int + gbp_search_query: dict[str, Any] + + +class HybridSessionAuth: + """Requests session with Playwright cookie refresh fallback. + + Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot + cookies when IAAI returns a challenge or an expected hidden payload is absent. + """ + + def __init__(self, settings: Settings) -> None: + self._settings = settings + self._thread_local = threading.local() + self._lock = threading.Lock() + self._refresh_lock = threading.Lock() + self._bootstrap_cookies_loaded = False + self._anonymous_bootstrap_attempted = False + self._refresh_generation = 0 + self._latest_refresh_cookies: list[dict[str, Any]] = [] + + def request( + self, + method: str, + url: str, + *, + timeout: int, + retries: int, + retry_backoff_ms: int, + headers: dict[str, str] | None = None, + data: Any | None = None, + json_body: Any | None = None, + expected_marker: str | None = None, + ) -> requests.Response: + session = self._get_session() + self._ensure_anonymous_session_bootstrap(session=session) + self._sync_session_with_latest_refresh(session) + last_error: Exception | None = None + refresh_attempts = 0 + attempt = 0 + max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts)) + + while attempt <= retries: + try: + request_started_at = time.perf_counter() + if self._settings.scraping_profile.verbose_http_logs: + logger.debug( + "HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s", + method, + url, + attempt + 1, + retries + 1, + timeout, + expected_marker, + ) + response = session.request( + method=method, + url=url, + headers=headers, + data=data, + json=json_body, + timeout=(min(10, max(1, timeout)), max(1, timeout)), + ) + if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400: + logger.debug( + "HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s", + method, + url, + response.status_code, + time.perf_counter() - request_started_at, + expected_marker, + ) + except requests.RequestException as exc: + last_error = exc + if attempt >= retries: + break + self._sleep_backoff(retry_backoff_ms, attempt) + attempt += 1 + continue + + if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries: + response.close() + self._sleep_backoff(retry_backoff_ms, attempt) + attempt += 1 + continue + + if is_challenge_response( + status_code=response.status_code, + body_text=response.text, + expected_marker=expected_marker, + ): + response.close() + if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts: + raise RuntimeError( + "IAAI challenge persisted after " + f"{refresh_attempts} Playwright refresh attempts for url={url}" + ) + refresh_attempts += 1 + logger.info( + "Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s", + url, + response.status_code, + expected_marker, + refresh_attempts, + max_refresh_attempts, + ) + self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session) + logger.info("Retrying HTTP request after Playwright refresh url=%s", url) + if refresh_attempts > 1: + self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1) + continue + + return response + + if last_error is not None: + raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error + raise RuntimeError(f"Request failed url={url} after retries") + + def persist_storage_state(self) -> None: + session = self._get_session() + with self._lock: + self._save_storage_state(session) + + def _get_session(self) -> requests.Session: + session = getattr(self._thread_local, "session", None) + if session is None: + session = requests.Session() + pool_size = max(20, int(self._settings.fetch_concurrency) * 2) + adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size) + session.mount("http://", adapter) + session.mount("https://", adapter) + session.headers.update( + { + "user-agent": DEFAULT_USER_AGENT, + "accept-language": "en-US,en;q=0.9", + "cache-control": "no-cache", + "pragma": "no-cache", + } + ) + if self._settings.proxy.enabled: + proxies = self._settings.proxy.to_requests_proxies() + if proxies: + session.proxies.update(proxies) + with self._lock: + self._bootstrap_session_cookies(session) + self._thread_local.session = session + self._thread_local.session_generation = 0 + self._sync_session_with_latest_refresh(session) + return session + + def _sync_session_with_latest_refresh(self, session: requests.Session) -> None: + with self._lock: + latest_generation = self._refresh_generation + session_generation = getattr(self._thread_local, "session_generation", 0) + if latest_generation <= session_generation or not self._latest_refresh_cookies: + return + cookies = list(self._latest_refresh_cookies) + self._apply_cookies_to_session(session, cookies) + self._thread_local.session_generation = latest_generation + + def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None: + if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled: + return + if self._session_has_iaai_cookies(session): + self._anonymous_bootstrap_attempted = True + return + with self._lock: + if self._anonymous_bootstrap_attempted: + return + self._anonymous_bootstrap_attempted = True + logger.info("No IAAI cookies preloaded. Attempting anonymous session bootstrap via Playwright.") + try: + self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session) + except Exception as exc: + logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc) + + @staticmethod + def _session_has_iaai_cookies(session: requests.Session) -> bool: + for item in session.cookies: + domain = str(getattr(item, "domain", "") or "") + if not domain or "iaai.com" in domain.lower(): + return True + return False + + def _bootstrap_session_cookies(self, session: requests.Session) -> None: + if self._bootstrap_cookies_loaded: + return + self._load_storage_state_cookies(session) + self._bootstrap_cookies_loaded = True + + def _load_storage_state_cookies(self, session: requests.Session) -> None: + tokens_file = self._settings.tokens_file + if not tokens_file: + return + path = __import__("pathlib").Path(tokens_file) + if not path.exists(): + return + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + logger.warning("Failed to read storage state file '%s': %s", path, exc) + return + cookies = payload.get("cookies") if isinstance(payload, dict) else None + if not isinstance(cookies, list): + return + applied = 0 + for item in cookies: + if not isinstance(item, dict): + continue + name = parse_text(item.get("name")) + value = parse_text(item.get("value")) + if not name or value is None: + continue + domain = parse_text(item.get("domain")) or ".iaai.com" + cookie_path = parse_text(item.get("path")) or "/" + expires = parse_int(item.get("expires")) + session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires) + applied += 1 + if applied: + logger.info("Loaded %s cookies from storage state", applied) + + def _refresh_session_via_playwright( + self, + *, + expected_marker: str | None = None, + session: requests.Session | None = None, + ) -> None: + target_session = session or self._get_session() + with self._lock: + baseline_generation = self._refresh_generation + + with self._refresh_lock: + with self._lock: + if self._refresh_generation > baseline_generation and self._latest_refresh_cookies: + self._apply_cookies_to_session(target_session, self._latest_refresh_cookies) + self._thread_local.session_generation = self._refresh_generation + return + + logger.info("IAAI session challenge detected. Refreshing session via Playwright.") + cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker) + logger.info("Playwright refresh returned %d cookies", len(cookies)) + self._apply_cookies_to_session(target_session, cookies) + logger.info("Playwright cookies applied to requests session") + + with self._lock: + self._refresh_generation += 1 + self._latest_refresh_cookies = list(cookies) + self._anonymous_bootstrap_attempted = True + refreshed_generation = self._refresh_generation + self._thread_local.session_generation = refreshed_generation + logger.info("Playwright refresh completed generation=%s", refreshed_generation) + + def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]: + from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + from playwright.sync_api import sync_playwright + + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=self._settings.headless) + try: + context = browser.new_context( + locale=self._settings.fingerprint.locale, + viewport={"width": 1366, "height": 768}, + user_agent=DEFAULT_USER_AGENT, + proxy=self._settings.proxy.to_playwright_dict(), + ) + page = context.new_page() + home_target = self._settings.home_url + filtered_urls = self._settings.listing.filtered_search_urls + target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars") + timeout_ms = max(30_000, self._settings.default_timeout_ms) + logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER) + page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms) + self._accept_cookie_banner(page) + page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms) + self._accept_cookie_banner(page) + self._wait_until_non_challenge( + page=page, + target=target, + timeout_ms=timeout_ms, + expected_marker=expected_marker or LISTING_MARKER, + ) + cookies = context.cookies() + logger.info("Playwright context returned %d cookies", len(cookies)) + except PlaywrightTimeoutError as exc: + raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc + finally: + try: + browser.close() + except Exception as exc: + logger.info("Playwright browser close failed after cookie refresh: %s", exc) + + if not isinstance(cookies, list) or not cookies: + raise RuntimeError("Playwright refresh did not return cookies") + return [cookie for cookie in cookies if isinstance(cookie, dict)] + + @staticmethod + def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None: + session.cookies.clear() + for cookie in cookies: + name = parse_text(cookie.get("name")) + value = parse_text(cookie.get("value")) + if not name or value is None: + continue + domain = parse_text(cookie.get("domain")) or ".iaai.com" + cookie_path = parse_text(cookie.get("path")) or "/" + expires = parse_int(cookie.get("expires")) + session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires) + + @staticmethod + def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None: + poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS)) + navigation_error_count = 0 + for poll_index in range(PLAYWRIGHT_REFRESH_POLLS): + try: + page.wait_for_load_state("domcontentloaded", timeout=poll_ms) + except Exception: + pass + page.wait_for_timeout(poll_ms) + body: str | None = None + for _ in range(3): + try: + body = page.content() + break + except Exception as exc: + message = str(exc).lower() + if "page.content" not in message or "navigating and changing the content" not in message: + raise + navigation_error_count += 1 + page.wait_for_timeout(max(200, poll_ms // 4)) + if body is not None and not is_challenge_response( + status_code=200, + body_text=body, + expected_marker=expected_marker, + ): + logger.info( + "Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s", + target, + poll_index + 1, + PLAYWRIGHT_REFRESH_POLLS, + expected_marker, + ) + return + try: + logger.info( + "Playwright session refresh still waiting target=%s poll=%s/%s marker=%s", + target, + poll_index + 1, + PLAYWRIGHT_REFRESH_POLLS, + expected_marker, + ) + page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms) + except Exception: + pass + raise RuntimeError( + "Playwright refresh completed but challenge page is still active " + f"(navigation_content_errors={navigation_error_count})" + ) + + @staticmethod + def _accept_cookie_banner(page: Any) -> None: + for selector in COOKIE_ACCEPT_SELECTORS: + try: + locator = page.locator(selector).first + if locator.count() == 0 or not locator.is_visible(timeout=500): + continue + locator.click(timeout=2_000) + page.wait_for_timeout(250) + return + except Exception: + continue + + def _save_storage_state(self, session: requests.Session) -> None: + tokens_file = self._settings.tokens_file + if not tokens_file: + return + path = __import__("pathlib").Path(tokens_file) + cookies: list[dict[str, Any]] = [] + for cookie in session.cookies: + payload: dict[str, Any] = { + "name": cookie.name, + "value": cookie.value, + "domain": cookie.domain or ".iaai.com", + "path": cookie.path or "/", + "httpOnly": False, + "secure": bool(cookie.secure), + "sameSite": "Lax", + } + if cookie.expires is not None: + payload["expires"] = int(cookie.expires) + cookies.append(payload) + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8") + except PermissionError as exc: + logger.info("Cannot persist IAAI storage state to '%s': %s", path, exc) + except OSError as exc: + logger.info("Failed to persist IAAI storage state to '%s': %s", path, exc) + + @staticmethod + def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None: + if retry_backoff_ms <= 0: + return + time.sleep(retry_backoff_ms * (2**attempt) / 1000) + + +class IAAIFastClient: + def __init__(self, settings: Settings) -> None: + self._settings = settings + self._auth = HybridSessionAuth(settings) + + def persist_session_state(self) -> None: + self._auth.persist_storage_state() + + def iter_listing_vehicles( + self, + *, + listing_start_url: str | None = None, + make: str | None = None, + max_pages: int | None = None, + ) -> Iterator[FastListingVehicle]: + seen_inventory_ids: set[str] = set() + scope_paths = resolve_listing_scope_paths( + listing_start_url=listing_start_url or "", + brands={make} if make else set(), + ) + for scope_path in scope_paths: + first_page_html = self._fetch_listing_first_page(scope_path) + search_scope_path = build_search_scope_path_from_html(first_page_html) + if search_scope_path and search_scope_path != scope_path: + logger.info( + "Resolved listing scope to fast Search URL: scope=%s search_scope=%s", + scope_path, + search_scope_path, + ) + scope_path = search_scope_path + first_page = parse_listing_page(first_page_html) + for vehicle in first_page.vehicles: + if vehicle.inventory_id in seen_inventory_ids: + continue + seen_inventory_ids.add(vehicle.inventory_id) + yield vehicle + + page_size = max(1, first_page.page_size) + total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size)) + if max_pages is not None and max_pages > 0: + total_pages = min(total_pages, max_pages) + gbp_search_query = first_page.gbp_search_query + for page_number in range(2, total_pages + 1): + page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size) + parsed_page = parse_listing_page(page_html) + gbp_search_query = parsed_page.gbp_search_query + for vehicle in parsed_page.vehicles: + if vehicle.inventory_id in seen_inventory_ids: + continue + seen_inventory_ids.add(vehicle.inventory_id) + yield vehicle + + def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]: + escaped_id = quote(inventory_id, safe="~") + url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}") + response = self._auth.request( + "GET", + url, + timeout=max(1, self._settings.fast_path_timeout_ms // 1000), + retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries), + retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), + headers={"accept": "text/html,application/xhtml+xml"}, + expected_marker=DETAIL_MARKER, + ) + with response: + if response.status_code >= 400: + raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}") + return parse_product_details_vm(response.text) + + def fetch_vehicle_detail_html(self, inventory_id: str) -> str: + escaped_id = quote(inventory_id, safe="~") + url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}") + response = self._auth.request( + "GET", + url, + timeout=max(1, self._settings.fast_path_timeout_ms // 1000), + retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries), + retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), + headers={"accept": "text/html,application/xhtml+xml"}, + expected_marker=DETAIL_MARKER, + ) + with response: + if response.status_code >= 400: + raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}") + return response.text + + def _fetch_listing_first_page(self, scope_path: str) -> str: + url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/")) + response = self._auth.request( + "GET", + url, + timeout=max(1, self._settings.fast_path_timeout_ms // 1000), + retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries), + retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), + headers={"accept": "text/html,application/xhtml+xml"}, + expected_marker=LISTING_MARKER, + ) + with response: + if response.status_code >= 400: + raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}") + return response.text + + def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str: + query_payload = dict(gbp_search_query) + query_payload["CurrentPage"] = page_number + query_payload["PageSize"] = page_size + search_url = urljoin(self._settings.home_url, "Search") + common_headers = { + "accept": "text/html,application/xhtml+xml,*/*", + "x-requested-with": "XMLHttpRequest", + } + attempts: list[tuple[dict[str, str], Any, Any]] = [ + ({**common_headers, "content-type": "application/json"}, None, query_payload), + ({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}), + ({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None), + ({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None), + ] + attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))] + last_error: Exception | None = None + for headers, data, json_body in attempts: + try: + response = self._auth.request( + "POST", + search_url, + timeout=max(1, self._settings.fast_path_timeout_ms // 1000), + retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries), + retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)), + headers=headers, + data=data, + json_body=json_body, + expected_marker=LISTING_MARKER, + ) + with response: + if response.status_code >= 400: + raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}") + body = response.text + if LISTING_MARKER not in body: + raise RuntimeError("Listing page response does not include GBPSearchQuery") + return body + except Exception as exc: + last_error = exc + continue + if last_error is not None: + raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error + raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}") + + +def build_brand_scope_paths(brands: set[str]) -> list[str]: + if not brands: + return ["/Vehiclelisting/Cars"] + paths: list[str] = [] + for brand in sorted(brands): + raw = brand.strip() + if not raw: + continue + override = BRAND_SCOPE_OVERRIDES.get(raw.upper()) + if override: + if override not in paths: + paths.append(override) + continue + slug_hyphen = quote(raw.replace(" ", "-"), safe="-") + slug_raw = quote(raw, safe="") + for slug in (slug_hyphen, slug_raw): + path = f"/Vehiclelisting/Cars/{slug}" + if path not in paths: + paths.append(path) + return paths or ["/Vehiclelisting/Cars"] + + +def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]: + explicit_scope = listing_start_url.strip() + if explicit_scope: + if explicit_scope.lower().startswith(("http://", "https://", "/")): + return [explicit_scope] + return [f"/Search?url={explicit_scope}"] + return build_brand_scope_paths(brands) + + +def build_search_scope_path_from_html(html_text: str) -> str | None: + tiny_url = parse_attribute_value(html_text, "data-tinyurl") + if tiny_url: + return f"/Search?url={tiny_url}" + + data_query_raw = parse_attribute_value(html_text, "data-query") + if data_query_raw: + try: + data_query = json.loads(data_query_raw) + except (json.JSONDecodeError, ValueError, TypeError): + data_query = None + if isinstance(data_query, dict): + url_value = parse_text(data_query.get("Url")) + if url_value: + return f"/Search?url={url_value}" + return None + + +def parse_listing_page(html_text: str) -> FastListingPage: + gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery") + vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails") + result_count_raw = parse_hidden_input_value(html_text, "ResultCount") + page_size_raw = parse_hidden_input_value(html_text, "PageSize") + current_page_raw = parse_hidden_input_value(html_text, "CurrentPage") + if not gbp_raw: + raise RuntimeError("Listing page missing GBPSearchQuery") + if vehicle_raw is None: + raise RuntimeError("Listing page missing VehicleDetails") + gbp_payload = json.loads(gbp_raw) + if not isinstance(gbp_payload, dict): + raise RuntimeError("GBPSearchQuery payload is not object") + vehicle_payload = json.loads(vehicle_raw) + if not isinstance(vehicle_payload, list): + raise RuntimeError("VehicleDetails payload is not array") + + vehicles: list[FastListingVehicle] = [] + for item in vehicle_payload: + if not isinstance(item, dict): + continue + inventory_id = parse_text(item.get("Id")) + if not inventory_id: + continue + vehicles.append( + FastListingVehicle( + inventory_id=inventory_id, + tenant=parse_text(item.get("Tenant")), + auction_id=parse_text(item.get("ActnLnId")), + auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")), + inventory_status=parse_text(item.get("InventoryStatus")), + currency=parse_text(item.get("Currency")), + timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")), + timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")), + prebid_indicator=parse_bool(item.get("PreBidIndicator")), + buynow_indicator=parse_bool(item.get("BuyNowIndicator")), + ) + ) + return FastListingPage( + vehicles=vehicles, + result_count=parse_int(result_count_raw) or len(vehicles), + page_size=parse_int(page_size_raw) or max(1, len(vehicles)), + current_page=parse_int(current_page_raw) or 1, + gbp_search_query=gbp_payload, + ) + + +def parse_product_details_vm(html_text: str) -> dict[str, Any]: + match = re.search( + r"]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*", + html_text, + flags=re.DOTALL | re.IGNORECASE, + ) + if match is None: + raise RuntimeError("ProductDetailsVM script not found") + payload = json.loads(match.group(1)) + if not isinstance(payload, dict): + raise RuntimeError("ProductDetailsVM root is not object") + return payload + + +def parse_hidden_input_value(html_text: str, input_id: str) -> str | None: + escaped_id = re.escape(input_id) + patterns = ( + rf"]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"", + rf"]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'", + ) + for pattern in patterns: + match = re.search(pattern, html_text, flags=re.IGNORECASE) + if match is not None: + return html.unescape(match.group(1)) + return None + + +def parse_attribute_value(html_text: str, attribute_name: str) -> str | None: + escaped_name = re.escape(attribute_name) + patterns = ( + rf"\b{escaped_name}=\"([^\"]*)\"", + rf"\b{escaped_name}='([^']*)'", + ) + for pattern in patterns: + match = re.search(pattern, html_text, flags=re.IGNORECASE) + if match is not None: + value = html.unescape(match.group(1)).strip() + return value or None + return None + + +def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]: + seen_fullres: set[str] = set() + images: list[dict[str, str | int]] = [] + for index, item in enumerate(image_keys): + if not isinstance(item, dict): + continue + key = parse_text(item.get("k")) + if key is None: + continue + width = parse_int(item.get("w")) or 1600 + height = parse_int(item.get("h")) or 1200 + if width <= 0: + width = 1600 + if height <= 0: + height = 1200 + order_index = parse_int(item.get("i")) + if order_index is None: + order_index = parse_int(item.get("in")) + if order_index is None: + order_index = index + preview_width = min(640, width) + preview_height = max(1, int(round(height * (preview_width / width)))) + escaped_key = quote(key, safe="~") + fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}" + preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}" + if fullres in seen_fullres: + continue + seen_fullres.add(fullres) + images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview}) + images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image")))) + return images + + +def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool: + if status_code in {401, 403}: + return True + if _expected_marker_present(body_text=body_text, expected_marker=expected_marker): + return False + lowered = (body_text or "").lower() + if any(marker in lowered for marker in CHALLENGE_MARKERS): + return True + if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker): + if " bool: + if not expected_marker: + return False + if expected_marker in body_text: + return True + if '"' in expected_marker and expected_marker.replace('"', "'") in body_text: + return True + if "'" in expected_marker and expected_marker.replace("'", '"') in body_text: + return True + marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker) + if marker_match is None: + return False + marker_id = re.escape(marker_match.group(1)) + return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE)) + + +def parse_text(value: Any) -> str | None: + if isinstance(value, str): + text = value.strip() + return text if text else None + return None + + +def parse_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"true", "1", "yes", "on"} + if isinstance(value, (int, float)) and not isinstance(value, bool): + return value != 0 + return False + + +def parse_int(value: Any) -> int | None: + if value is None or isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(round(value)) + if isinstance(value, str): + text = value.strip() + if not text: + return None + normalized = text.replace(",", "").replace(" ", "").replace("$", "") + match = re.search(r"-?\d+(?:\.\d+)?", normalized) + if match is None: + return None + try: + return int(round(float(match.group(0)))) + except ValueError: + return None + return None diff --git a/iaai_scraper/browser/listing.py b/iaai_scraper/browser/listing.py new file mode 100644 index 0000000..55545cc --- /dev/null +++ b/iaai_scraper/browser/listing.py @@ -0,0 +1,714 @@ +import logging +import re +import time +from dataclasses import asdict, dataclass, field +from typing import Any +from urllib.parse import urljoin + +from playwright.sync_api import Page + +from .pace import HumanPacer +from ..core.config import Settings +from ..core.utils import first_non_empty + +logger = logging.getLogger("iaai_scraper.listing") +VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) +VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']" +COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = ( + "button:has-text('Accept All')", + "button:has-text('Accept all')", + "button:has-text('I Agree')", + "button:has-text('Agree')", + "button:has-text('Only necessary')", + "button:has-text('Только необходимые')", + "button:has-text('Принять все')", + "[id*='accept']", + "[class*='accept']", +) + + +@dataclass(slots=True) +class ListingVehicleLink: + href: str + title: str = "" + lot_number: str | None = None + + +@dataclass(slots=True) +class ListingPageResult: + source_url: str + page_number: int + vehicle_links: list[ListingVehicleLink] = field(default_factory=list) + pagination_available: bool = False + next_page_detected: bool = False + + +class ListingCollector: + _NEXT_PAGE_SELECTORS: tuple[str, ...] = ( + "a[aria-label*='Next']", + "button[aria-label*='Next']", + "a[aria-label*='next']", + "button[aria-label*='next']", + "a[title*='Next']", + "button[title*='Next']", + "a[title*='next']", + "button[title*='next']", + "a[rel='next']", + "link[rel='next']", + "a.pagination-next", + "button.pagination-next", + "a.next", + "button.next", + "a:has-text('Next')", + "button:has-text('Next')", + "a:has-text('NEXT')", + "button:has-text('NEXT')", + "a:has-text('›')", + "button:has-text('›')", + "a:has-text('»')", + "button:has-text('»')", + "a:has(img[src*='icon-arrow-right'])", + "button:has(img[src*='icon-arrow-right'])", + "a:has(img[src*='arrow-right'])", + "button:has(img[src*='arrow-right'])", + ) + + def __init__(self, settings: Settings, pacer: HumanPacer) -> None: + self.settings = settings + self.pacer = pacer + + @staticmethod + def _get_current_page_number(page: Page) -> int | None: + try: + value = page.evaluate( + """ + () => { + const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + if (current) { + return parseInt((current.textContent || '').trim(), 10); + } + const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i); + return match ? parseInt(match[1], 10) : null; + } + """ + ) + return int(value) if value is not None else None + except Exception: + return None + + @staticmethod + def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool: + if old_first_href: + try: + page.wait_for_function( + f"""() => {{ + const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\"); + return a && a.getAttribute('href') !== '{old_first_href}'; + }}""", + timeout=12000, + ) + return True + except Exception: + pass + + if expected_page_number is not None: + current_page = ListingCollector._get_current_page_number(page) + if current_page == expected_page_number: + return True + + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) + except Exception: + pass + + current_page = ListingCollector._get_current_page_number(page) + if expected_page_number is not None and current_page == expected_page_number: + return True + + return not old_first_href + + @staticmethod + def has_page_number(page: Page, target_page_number: int) -> bool: + try: + return bool(page.evaluate( + """ + (targetPageNumber) => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); + return controls.some((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber; + }); + } + """, + target_page_number, + )) + except Exception: + return False + + def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None: + url = url_override or self.settings.listing.cars_url + logger.info("Opening cars listing page: %s", url) + last_err = None + for attempt in range(3): + try: + page.goto(url, wait_until="commit", timeout=60_000) + last_err = None + break + except Exception as e: + last_err = e + logger.warning("goto listing attempt %d failed: %s", attempt + 1, e) + # Небольшой backoff при ошибках открытия листинга. + time.sleep(5 * (attempt + 1)) + if last_err: + logger.warning("All goto attempts failed, trying JS navigation") + try: + page.evaluate(f"window.location.href = '{url}'") + except Exception: + pass + # Быстрая проверка готовности страницы. + try: + page.wait_for_load_state("domcontentloaded", timeout=12_000) + except Exception: + pass + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) + logger.info("Listing page URL: %s", page.url) + self.pacer.after_listing_open() + + def _accept_cookie_banner(self, page: Page) -> None: + for selector in COOKIE_ACCEPT_SELECTORS: + locator = page.locator(selector).first + try: + if locator.count() == 0: + continue + if not locator.is_visible(timeout=500): + continue + locator.click(timeout=2_000) + logger.info("Accepted cookie banner using selector: %s", selector) + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except Exception: + pass + return + except Exception: + continue + + def _wait_for_listing_content(self, page: Page) -> None: + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000) + return + except Exception: + pass + + # Fallback: React/SSR разметка может появиться не сразу, даже если ещё нет в DOM. + try: + page.wait_for_function( + """() => { + const html = document.documentElement?.innerHTML || ''; + const text = document.body?.innerText || ''; + return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text); + }""", + timeout=12_000, + ) + except Exception: + # Короткая пауза вместо длинного sleep. + time.sleep(1.0) + + def apply_filters( + self, + page: Page, + make: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, str | int | None]: + applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None} + if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): + applied["make"] = make + self.pacer.after_filter_action() + if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): + applied["model"] = model + self.pacer.after_filter_action() + if year_min is not None or year_max is not None: + if self._apply_year_range(page, year_min, year_max): + applied["year_min"] = year_min + applied["year_max"] = year_max + self.pacer.after_filter_action() + return applied + + def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool: + """Заполняет поля фильтра Year и нажимает Apply Year.""" + if year_min is None and year_max is None: + return False + try: + success = page.evaluate( + """([yearMin, yearMax]) => { + const inputs = Array.from(document.querySelectorAll('input')); + const yearInputs = inputs.filter(inp => { + const v = parseInt(inp.value, 10); + return !isNaN(v) && v >= 1900 && v <= 2100; + }); + if (yearInputs.length < 2) return false; + yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value)); + const setVal = (el, val) => { + const setter = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, 'value' + ).set; + setter.call(el, String(val)); + el.dispatchEvent(new Event('input', {bubbles: true})); + el.dispatchEvent(new Event('change', {bubbles: true})); + }; + if (yearMin !== null) setVal(yearInputs[0], yearMin); + if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax); + const container = yearInputs[0].closest( + '[class*="filter"], [class*="year"], section, fieldset' + ) || yearInputs[0].parentElement.parentElement; + if (container) { + const btn = Array.from(container.querySelectorAll( + 'button, a, [role="button"], span[class*="apply"]' + )).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent)); + if (btn) { btn.click(); return true; } + } + yearInputs[yearInputs.length - 1].dispatchEvent( + new KeyboardEvent('keydown', { + key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true + }) + ); + return true; + }""", + [year_min, year_max], + ) + if success: + try: + page.wait_for_load_state("domcontentloaded", timeout=15_000) + except Exception: + pass + self._wait_for_listing_content(page) + logger.info("Applied year range filter: %s — %s", year_min, year_max) + return True + except Exception as exc: + logger.warning("Failed to apply year range filter: %s", exc) + return False + + def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: + # Считываем ссылки одним проходом по DOM. + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) + try: + raw_items = page.eval_on_selector_all( + "a[href], [data-href], [href]", + """ + (nodes) => nodes.map((node) => ({ + href: + node.getAttribute('href') || + node.getAttribute('data-href') || + node.getAttribute('data-url') || + '', + title: node.getAttribute('title') || node.getAttribute('aria-label') || '', + text: (node.textContent || '').trim(), + })) + """, + ) + except Exception as exc: + logger.warning("collect_current_page failed on page %d: %s", page_number, exc) + raw_items = [] + total = min(len(raw_items), self.settings.listing.page_link_limit) + links: list[ListingVehicleLink] = [] + seen: set[str] = set() + for idx in range(total): + item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} + href = str(item.get("href") or "") + match = VEHICLE_HREF_RE.search(href) + if not match: + continue + lot_number = match.group(1) + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" + links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + # Fallback: на IAAI ссылки иногда не рендерятся как , + # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). + if not links: + try: + page.wait_for_timeout(1_500) + except Exception: + pass + try: + html = page.content() + except Exception as exc: + logger.debug("page.content() failed on page %d: %s", page_number, exc) + html = "" + + for absolute, lot_number in self._extract_vehicle_links_from_html(html): + if absolute in seen: + continue + seen.add(absolute) + links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + if links: + logger.info( + "Page %d: recovered %d vehicle links from HTML fallback", + page_number, + len(links), + ) + next_page_detected = self._has_next_page(page) + return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) + + def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]: + if not html: + return [] + + # Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US" + normalized = html.replace("\\/", "/") + found: list[tuple[str, str]] = [] + seen: set[str] = set() + + for match in VEHICLE_HREF_RE.finditer(normalized): + lot_number = match.group(1) + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + found.append((absolute, lot_number)) + if len(found) >= self.settings.listing.page_link_limit: + break + + return found + + def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool: + # Запоминаем первую ссылку текущей страницы для определения смены контента. + old_first_href = "" + try: + first_link = page.locator(VEHICLE_LINK_SELECTOR).first + if first_link.count() > 0: + old_first_href = first_link.get_attribute("href") or "" + except Exception: + pass + + for selector in self._NEXT_PAGE_SELECTORS: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower() + aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower() + classes = (locator.get_attribute("class", timeout=1500) or "").lower() + except Exception: + continue + if disabled or aria_disabled == "true" or "disabled" in classes: + continue + try: + self.pacer.move_mouse_to(page, locator) + locator.click(timeout=8000) + except Exception: + continue + + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + + # Fallback для IAAI: пагинация часто рендерится как набор номеров страниц + # + стрелка с иконкой, без явного текста Next. + try: + clicked = bool(page.evaluate( + """ + () => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const disabled = (el) => { + if (!el) return true; + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + }; + + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')) + .filter((el) => visible(el) && !disabled(el)); + + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + + if (current) { + const currentPage = parseInt((current.textContent || '').trim(), 10); + const nextNumber = controls.find((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; + }); + if (nextNumber) { + nextNumber.click(); + return true; + } + } + + const iconNext = controls.find((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + }); + + if (iconNext) { + iconNext.click(); + return true; + } + + return false; + } + """ + )) + if clicked: + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("Numeric/icon pagination fallback failed: %s", exc) + + # JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту. + try: + clicked = bool(page.evaluate( + """ + () => { + const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]')); + for (const el of candidates) { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); + const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + if (!disabled && visible && looksNext) { + el.click(); + return true; + } + } + return false; + } + """ + )) + if clicked: + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("JS next-page fallback failed: %s", exc) + + logger.warning("Could not navigate to next page from %s", page.url) + return False + + def collect_listing_links( + self, + page: Page, + *, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, + ) -> dict[str, Any]: + self.open_cars_listing(page) + applied_filters = self.apply_filters(page, make=make, model=model) + started_at = time.perf_counter() + truncated_by_time_budget = False + pages: list[dict[str, object]] = [] + all_links: list[str] = [] + early_stopped = False + threshold = self.settings.listing.early_stop_threshold + + for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): + if max_duration_seconds is not None and max_duration_seconds > 0: + elapsed = time.perf_counter() - started_at + if elapsed >= max_duration_seconds: + truncated_by_time_budget = True + logger.warning( + "Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs", + page_number, + elapsed, + max_duration_seconds, + ) + break + page_result = self.collect_current_page(page, page_number=page_number) + pages.append({ + "page_number": page_result.page_number, + "source_url": page_result.source_url, + "links_found": len(page_result.vehicle_links), + "vehicle_links": [asdict(item) for item in page_result.vehicle_links], + "next_page_detected": page_result.next_page_detected, + }) + for item in page_result.vehicle_links: + if item.href not in all_links: + all_links.append(item.href) + if len(all_links) >= self.settings.listing.max_vehicles_per_run: + break + + # Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла. + # Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице). + if ( + known_origin_ids is not None + and threshold > 0.0 + and page_result.vehicle_links + ): + page_known = sum( + 1 for item in page_result.vehicle_links + if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids + ) + page_new = len(page_result.vehicle_links) - page_known + ratio = page_known / len(page_result.vehicle_links) + # Останавливаемся только если нет новых И порог превышен + if ratio >= threshold and page_new == 0: + logger.info( + "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", + page_number, ratio * 100, page_known, + len(page_result.vehicle_links), threshold * 100, + ) + early_stopped = True + break + elif page_new > 0 and ratio >= threshold: + logger.info( + "Page %d: %.0f%% known but %d new found — продолжаем", + page_number, ratio * 100, page_new, + ) + + if ( + len(all_links) >= self.settings.listing.max_vehicles_per_run + or self.settings.listing.collect_current_page_only + or not self.settings.listing.include_pagination + or not page_result.next_page_detected + ): + break + if not self.go_to_next_page(page): + break + + return { + "listing_url": self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": len(pages), + "vehicles_collected": len(all_links), + "vehicle_urls": all_links, + "early_stopped": early_stopped, + "truncated_by_time_budget": truncated_by_time_budget, + "pages": pages, + "strategy": { + "sequential": True, + "collect_current_page_only": self.settings.listing.collect_current_page_only, + "include_pagination": self.settings.listing.include_pagination, + "max_pages_per_run": self.settings.listing.max_pages_per_run, + "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run, + "early_stop_threshold": threshold, + }, + } + + @staticmethod + def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + locator.click() + locator.fill(value) + page.keyboard.press("Enter") + try: + page.wait_for_load_state("domcontentloaded", timeout=15000) + except Exception: + pass + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) + except Exception: + pass + return True + except Exception: + continue + return False + + @staticmethod + def _has_next_page(page: Page) -> bool: + for selector in ListingCollector._NEXT_PAGE_SELECTORS: + locator = page.locator(selector) + count = locator.count() + if count == 0: + continue + if not hasattr(locator, "nth"): + return True + # Проверяем, что хотя бы один элемент не disabled. + # Disabled "Next" на последней странице не означает наличия следующей. + for i in range(min(count, 3)): + try: + el = locator.nth(i) + disabled_attr = el.get_attribute("disabled", timeout=300) + aria_disabled = el.get_attribute("aria-disabled", timeout=300) + cls = (el.get_attribute("class", timeout=300) or "").lower() + if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls: + return True + except Exception: + continue + try: + return bool(page.evaluate( + """ + () => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => { + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + return !disabled && visible(el); + }); + + const hasExplicitNext = controls.some((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text); + }); + + if (hasExplicitNext) { + return true; + } + + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + + if (!current) { + return false; + } + + const currentPage = parseInt((current.textContent || '').trim(), 10); + return controls.some((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; + }); + } + """ + )) + except Exception: + return False + return False diff --git a/iaai_scraper/browser/network.py b/iaai_scraper/browser/network.py new file mode 100644 index 0000000..0674976 --- /dev/null +++ b/iaai_scraper/browser/network.py @@ -0,0 +1,130 @@ +import json +import logging +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlparse + +from playwright.sync_api import Page, Request, Response + +from ..core.config import Settings + +logger = logging.getLogger("iaai_scraper.network") + + +@dataclass +class NetworkCapture: + # Перехватчик сетевых запросов. + + settings: Settings + requests: list[dict[str, Any]] = field(default_factory=list) + json_responses: list[dict[str, Any]] = field(default_factory=list) + _seen_req: set[str] = field(default_factory=set) + _seen_resp: set[str] = field(default_factory=set) + _origin: str | None = None + _page: Any = field(default=None) + + def attach(self, page: Page, origin_url: str | None = None) -> None: + # Снимаем старые подписки перед повторным attach. + try: + page.remove_listener("request", self._on_request) + page.remove_listener("response", self._on_response) + except Exception: + pass + # Подписка на сетевые события + try: + self._origin = urlparse(origin_url or page.url).netloc.lower() or None + except Exception: + self._origin = None + self._page = page + page.on("request", self._on_request) + page.on("response", self._on_response) + + def _is_same_origin(self, url: str) -> bool: + # Фильтр по домену + if not self.settings.capture.capture_same_origin_only or not self._origin: + return True + netloc = urlparse(url).netloc.lower() + return netloc == self._origin or netloc.endswith(".iaai.com") + + def _on_request(self, request: Request) -> None: + # Берём только xhr/fetch + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(request.url): + return + if len(self.requests) >= self.settings.capture.max_requests: + return + key = f"{request.method}:{request.url}:{request.post_data or ''}" + # Убираем дубли + if key in self._seen_req: + return + self._seen_req.add(key) + self.requests.append({ + "url": request.url, "method": request.method, + "resource_type": request.resource_type, "post_data": request.post_data, + }) + + def _on_response(self, response: Response) -> None: + # Сохраняем JSON + request = response.request + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(response.url): + return + if len(self.json_responses) >= self.settings.capture.max_json_responses: + return + if not self._is_json(response): + return + key = f"{request.method}:{response.url}:{response.status}" + if key in self._seen_resp: + return + self._seen_resp.add(key) + try: + payload = response.json() + except Exception: + try: + payload = json.loads(response.text()) + except Exception: + return + self.json_responses.append({ + "url": response.url, "status": response.status, + "request_method": request.method, "post_data": request.post_data, + "resource_type": request.resource_type, "payload": payload, + "category": self._categorize(response.url), + }) + + @staticmethod + def _is_json(response: Response) -> bool: + ct = (response.headers.get("content-type") or "").lower() + if "application/json" in ct or "+json" in ct: + return True + url = response.url.lower() + return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) + + @staticmethod + def _categorize(url: str) -> str: + # Простая категория URL + low = url.lower() + mapping = { + "images": ["image", "media", "photos", "gallery"], + "bids": ["bid", "offer", "buy-now", "buynow"], + "auction": ["auction", "sale", "lane", "branch"], + "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], + "documents": ["title", "document", "report"], + } + for cat, markers in mapping.items(): + if any(m in low for m in markers): + return cat + return "other" + + def export(self) -> dict[str, Any]: + responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) + return { + "requests": self.requests, + "json_responses": responses, + "capture_limits": { + "same_origin_only": self.settings.capture.capture_same_origin_only, + "max_requests": self.settings.capture.max_requests, + "max_json_responses": self.settings.capture.max_json_responses, + }, + } diff --git a/iaai_scraper/browser/pace.py b/iaai_scraper/browser/pace.py new file mode 100644 index 0000000..10c2551 --- /dev/null +++ b/iaai_scraper/browser/pace.py @@ -0,0 +1,46 @@ +import random +import time + +from playwright.sync_api import Locator, Page + +from ..core.config import Settings + + +class HumanPacer: + # Случайные паузы между действиями. + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def pause(self, min_s: float, max_s: float) -> None: + if self.settings.pace.enabled: + time.sleep(random.uniform(min_s, max_s)) + + def after_listing_open(self) -> None: + self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) + + def after_filter_action(self) -> None: + self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) + + def before_vehicle_open(self) -> None: + self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) + + def after_vehicle_open(self) -> None: + self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) + + def between_vehicles(self) -> None: + self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) + + def after_page_change(self) -> None: + self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) + + def move_mouse_to(self, page: Page, locator: Locator) -> None: + try: + box = locator.bounding_box() + except Exception: + box = None + if not box: + return + x = box["x"] + box["width"] * random.uniform(0.2, 0.8) + y = box["y"] + box["height"] * random.uniform(0.2, 0.8) + page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py new file mode 100644 index 0000000..51ce894 --- /dev/null +++ b/iaai_scraper/cli.py @@ -0,0 +1,168 @@ +import argparse +from pathlib import Path + +from .core.config import Settings +from .core.utils import save_to_json +from .mobile_de import MobileDeClient, MobileDeScraper +from .scraper import IAAIScraper + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="mobile.de scraper CLI") + parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") + parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") + subparsers = parser.add_subparsers(dest="command", required=True) + + default_output_dir = Path("artifacts/json") + + subparsers.add_parser("init-db", help="Create DB tables") + + listing_parser = subparsers.add_parser("collect-listing", help="Deprecated IAAI command: collect vehicle URLs from listing page") + listing_parser.add_argument("--make", default=None) + listing_parser.add_argument("--model", default=None) + listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json")) + + scrape_parser = subparsers.add_parser("scrape-vehicle", help="Deprecated IAAI command: scrape a vehicle detail page") + scrape_parser.add_argument("vehicle_url") + scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json")) + + sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Deprecated IAAI command: scrape + upsert one vehicle") + sync_vehicle_parser.add_argument("vehicle_url") + sync_vehicle_parser.add_argument("--lane", default="iaai") + sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json")) + + sync_listing_parser = subparsers.add_parser("sync-listing", help="Deprecated IAAI command: collect listing + sync all vehicles") + sync_listing_parser.add_argument("--make", default=None) + sync_listing_parser.add_argument("--model", default=None) + sync_listing_parser.add_argument("--lane", default="iaai_cars") + sync_listing_parser.add_argument("--limit", type=int, default=None) + sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) + sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json")) + + mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages") + mobile_search_parser.add_argument("--page", type=int, default=1) + mobile_search_parser.add_argument("--max-pages", type=int, default=1) + mobile_search_parser.add_argument("--delay", type=float, default=0.7) + mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами") + mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500") + mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id") + mobile_search_parser.add_argument("--price-min", default=None) + mobile_search_parser.add_argument("--price-max", default=None) + mobile_search_parser.add_argument("--year-min", default=None) + mobile_search_parser.add_argument("--year-max", default=None) + mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json")) + + mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page") + mobile_detail_parser.add_argument("listing_id") + mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json")) + + mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages") + mobile_sync_search_parser.add_argument("--page", type=int, default=1) + mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1) + mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7) + mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars") + mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами") + mobile_sync_search_parser.add_argument("--make-id", default=None) + mobile_sync_search_parser.add_argument("--model-id", default=None) + mobile_sync_search_parser.add_argument("--price-min", default=None) + mobile_sync_search_parser.add_argument("--price-max", default=None) + mobile_sync_search_parser.add_argument("--year-min", default=None) + mobile_sync_search_parser.add_argument("--year-max", default=None) + mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json")) + + mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page") + mobile_sync_detail_parser.add_argument("listing_id") + mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars") + mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json")) + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + runtime_settings: Settings | None = None + if args.headless is not None or args.debug: + runtime_settings = Settings() + if args.headless is not None: + runtime_settings.headless = args.headless == "true" + if args.debug: + runtime_settings.log_level = "DEBUG" + + if args.command in {"search", "mobilede-search"}: + scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay)) + data = scraper.collect_search( + start_page=args.page, + max_pages=args.max_pages, + search_url=args.search_url, + make_id=args.make_id, + model_id=args.model_id, + price_min=args.price_min, + price_max=args.price_max, + year_min=args.year_min, + year_max=args.year_max, + ) + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + return + + if args.command in {"detail", "mobilede-detail"}: + scraper = MobileDeScraper() + data = scraper.collect_detail(args.listing_id) + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + return + + if args.command == "sync-search": + scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay)) + data = scraper.sync_search( + start_page=args.page, + max_pages=args.max_pages, + lane=args.lane, + search_url=args.search_url, + make_id=args.make_id, + model_id=args.model_id, + price_min=args.price_min, + price_max=args.price_max, + year_min=args.year_min, + year_max=args.year_max, + ) + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + return + + if args.command == "sync-detail": + scraper = MobileDeScraper() + data = scraper.sync_detail(args.listing_id, lane=args.lane) + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + return + + with IAAIScraper(runtime_settings) as scraper: + if args.command == "init-db": + data = scraper.init_db() + print(f"DB initialized: {data}") + return + elif args.command == "collect-listing": + data = scraper.collect_listing(make=args.make, model=args.model) + elif args.command == "scrape-vehicle": + data = scraper.scrape_vehicle_detail(args.vehicle_url) + elif args.command == "sync-vehicle": + data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) + else: + only_new = None if args.only_new is None else args.only_new == "true" + data = scraper.sync_listing( + make=args.make, + model=args.model, + lane=args.lane, + limit=args.limit, + only_new=only_new, + ) + + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + + +if __name__ == "__main__": + main() diff --git a/iaai_scraper/core/__init__.py b/iaai_scraper/core/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/core/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py new file mode 100644 index 0000000..07dab03 --- /dev/null +++ b/iaai_scraper/core/config.py @@ -0,0 +1,434 @@ +import json +import os +from dataclasses import dataclass, field +from pathlib import Path +from urllib.parse import quote, urlsplit, urlunsplit + +from dotenv import load_dotenv + +load_dotenv() + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +# Хелперы для чтения env-переменных с приведением типов + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + return value if value is not None else default + + +def _env_optional_str(name: str) -> str | None: + value = os.getenv(name) + if value is None: + return None + value = value.strip() + return value or None + + +def _env_path_str(name: str) -> str | None: + value = _env_optional_str(name) + if value is None: + return None + return str(Path(value).expanduser()) + + +def _env_bool(name: str, default: bool) -> bool: + fallback = "true" if default else "false" + return _env_str(name, fallback).strip().lower() in TRUE_VALUES + + +def _env_int(name: str, default: int) -> int: + return int(_env_str(name, str(default)).strip()) + + +def _env_float(name: str, default: float) -> float: + return float(_env_str(name, str(default)).strip()) + + +# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +# Конфиг перехвата сетевых запросов (лимиты на кол-во) + +@dataclass(slots=True) +class CaptureConfig: + capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True) + max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40) + max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30) + + +# Конфиг пауз между действиями (имитация человека) + +@dataclass(slots=True) +class HumanPaceConfig: + enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True) + after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5) + after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2) + after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5) + after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2) + before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) + before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) + after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05) + after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15) + between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05) + between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15) + after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8) + after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8) + + +# Конфиг сбора листинга (URL, лимиты страниц и машин) + +@dataclass(slots=True) +class ListingConfig: + cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") + filtered_search_url: str | None = _env_optional_str("IAAI_FILTERED_SEARCH_URL") + filtered_search_urls_raw: str | None = _env_optional_str("IAAI_FILTERED_SEARCH_URLS") + max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999) + max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000) + page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500) + include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True) + collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False) + # Порог раннего останова: если доля уже известных машин на странице >= этого значения, + # прекращаем листать — все новые машины уже найдены. 0 = отключено. + early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8) + # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин). + # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка + # или "runtime" для построения по runtime_config.filters.brands. + # Пустая строка = без сегментации (backward compatible). + listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "") + fast_segment_year_splits: bool = _env_bool("IAAI_FAST_SEGMENT_YEAR_SPLITS", True) + + @property + def filtered_search_urls(self) -> list[str]: + urls: list[str] = [] + if self.filtered_search_url and self.filtered_search_url.strip(): + urls.append(self.filtered_search_url.strip()) + if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip(): + raw = self.filtered_search_urls_raw.strip() + try: + parsed = json.loads(raw) + except (json.JSONDecodeError, ValueError): + parsed = None + if isinstance(parsed, list): + candidates = [str(item or "").strip() for item in parsed] + else: + candidates = [part.strip() for part in raw.replace("\n", ",").split(",")] + for candidate in candidates: + if candidate and candidate not in urls: + urls.append(candidate) + return urls + + +# Список брендов IAAI для автоматической сегментации. +# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким. +IAAI_DEFAULT_MAKES: tuple[str, ...] = ( + "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", + "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU", + "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI", + "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN", + "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE", + "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI", + "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI", + "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART", + "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI", + "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS", + "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE", +) + +# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов. +IAAI_PAGINATION_CEILING = 22_600 + +# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам. +_LARGE_MAKES: frozenset[str] = frozenset({ + "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", + "KIA", "DODGE", "JEEP", +}) +_YEAR_SPLITS: tuple[tuple[int, int], ...] = ( + (1900, 2012), + (2013, 2019), + (2020, 2027), +) + + +def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]: + """Строит сегменты по переданному списку брендов. + + Крупные бренды режутся по годовым диапазонам так же, как в ``auto``. + """ + segments: list[dict[str, str | int | None]] = [] + seen: set[str] = set() + for raw_make in makes: + make = str(raw_make or "").strip().upper() + if not make or make in seen: + continue + seen.add(make) + if make in _LARGE_MAKES: + for yr_min, yr_max in _YEAR_SPLITS: + segments.append({"make": make, "year_min": yr_min, "year_max": yr_max}) + else: + segments.append({"make": make, "year_min": None, "year_max": None}) + return segments + + +def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]: + """Строит HTTP-first сегменты без UI-фильтров годов. + + Это ближе к iaai-fast: один бренд = один hidden-payload HTTP обход. + Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля. + """ + segments: list[dict[str, str | int | None]] = [] + seen: set[str] = set() + for raw_make in makes: + make = str(raw_make or "").strip().upper() + if not make or make in seen: + continue + seen.add(make) + segments.append({"make": make, "year_min": None, "year_max": None}) + return segments + + +def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]: + """Парсит IAAI_LISTING_SEGMENTS в список сегментов. + + Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None). + Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES. + Крупные бренды автоматически разбиваются по диапазонам годов. + """ + raw = raw.strip() + if not raw: + return [] + if raw.lower() == "auto": + return build_listing_segments_for_makes(list(IAAI_DEFAULT_MAKES)) + try: + data = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return [] + if not isinstance(data, list): + return [] + segments = [] + for item in data: + if isinstance(item, str): + segments.append({"make": item.upper(), "year_min": None, "year_max": None}) + elif isinstance(item, dict): + segments.append({ + "make": str(item.get("make") or "").upper() or None, + "year_min": int(item["year_min"]) if item.get("year_min") is not None else None, + "year_max": int(item["year_max"]) if item.get("year_max") is not None else None, + }) + return segments + + +# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) + +@dataclass(slots=True) +class DatabaseConfig: + url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper") + echo: bool = _env_bool("IAAI_DATABASE_ECHO", False) + pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5) + max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10) + pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800) + auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False) + + +# --- Конфиг Redis (URL для Celery broker) --- + +@dataclass(slots=True) +class RedisConfig: + url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0") + socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) + socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) + health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) + + +# --- Конфиг Discovery (режим обнаружения, hourly batch) --- + +@dataclass(slots=True) +class DiscoveryConfig: + mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap") + hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh") + hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500) + always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", False) + + +# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = _env_str("CELERY_BROKER_URL", "") + result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") + task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) + task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) + task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600) + worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) + worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork") + worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) + broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) + beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) + beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None + batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) + parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8) + fetch_concurrency: int = _env_int("IAAI_FETCH_CONCURRENCY", _env_int("IAAI_PARALLEL_TABS", 8)) + parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False) + block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True) + + +@dataclass(slots=True) +class ScrapingProfileConfig: + name: str = _env_str("IAAI_SCRAPING_PROFILE", _env_str("IAAI_PROFILE", "stable")).strip().lower() + http_first: bool = _env_bool("IAAI_HTTP_FIRST", True) + browser_fallback_enabled: bool = _env_bool("IAAI_BROWSER_FALLBACK_ENABLED", True) + anonymous_bootstrap_enabled: bool = _env_bool("IAAI_ANONYMOUS_BOOTSTRAP_ENABLED", True) + verbose_http_logs: bool = _env_bool("IAAI_VERBOSE_HTTP_LOGS", False) + verbose_progress_logs: bool = _env_bool("IAAI_VERBOSE_PROGRESS_LOGS", False) + challenge_refresh_enabled: bool = _env_bool("IAAI_CHALLENGE_REFRESH_ENABLED", True) + challenge_refresh_attempts: int = _env_int("IAAI_CHALLENGE_REFRESH_ATTEMPTS", 3) + listing_post_attempts: int = _env_int("IAAI_LISTING_POST_ATTEMPTS", 4) + request_jitter_max_s: float = _env_float("IAAI_REQUEST_JITTER_MAX_S", 0.15) + detail_retries: int | None = _env_int("IAAI_DETAIL_RETRIES", -1) + listing_retries: int | None = _env_int("IAAI_LISTING_RETRIES", -1) + + def __post_init__(self) -> None: + if self.name == "fast": + if "IAAI_BROWSER_FALLBACK_ENABLED" not in os.environ: + self.browser_fallback_enabled = False + if "IAAI_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ: + self.anonymous_bootstrap_enabled = False + if "IAAI_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ: + self.challenge_refresh_attempts = 1 + if "IAAI_LISTING_POST_ATTEMPTS" not in os.environ: + self.listing_post_attempts = 2 + if "IAAI_REQUEST_JITTER_MAX_S" not in os.environ: + self.request_jitter_max_s = 0.0 + if "IAAI_DETAIL_RETRIES" not in os.environ: + self.detail_retries = 1 + if "IAAI_LISTING_RETRIES" not in os.environ: + self.listing_retries = 1 + else: + if self.detail_retries is not None and self.detail_retries < 0: + self.detail_retries = None + if self.listing_retries is not None and self.listing_retries < 0: + self.listing_retries = None + + +# --- Конфиг прокси (server, username, password) --- + +@dataclass(slots=True) +class ProxyConfig: + server: str | None = _env_optional_str("IAAI_PROXY_SERVER") + username: str | None = _env_optional_str("IAAI_PROXY_USERNAME") + password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD") + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + def to_requests_proxy_url(self) -> str | None: + if not self.server: + return None + if not self.username: + return self.server + + parts = urlsplit(self.server) + if not parts.scheme or not parts.hostname: + return self.server + + username = quote(self.username, safe="") + password = quote(self.password or "", safe="") + host = parts.hostname + if ":" in host and not host.startswith("["): + host = f"[{host}]" + if parts.port is not None: + host = f"{host}:{parts.port}" + netloc = f"{username}:{password}@{host}" + return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) + + def to_requests_proxies(self) -> dict[str, str] | None: + proxy_url = self.to_requests_proxy_url() + if not proxy_url: + return None + return {"http": proxy_url, "https": proxy_url} + + +# Главный объект настроек: собирает все блоки конфигурации + +@dataclass(slots=True) +class Settings: + home_url: str = "https://www.iaai.com/" + default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000) + network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400) + fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 15000) + fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1) + fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000) + max_retries: int = _env_int("IAAI_MAX_RETRIES", 3) + retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5) + retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0) + retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25) + headless: bool = _env_bool("IAAI_HEADLESS", True) + browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto") + log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO") + log_file: str | None = _env_optional_str("IAAI_LOG_FILE") + enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True) + sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False) + raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON") + tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE") + runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE") + scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60) + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + capture: CaptureConfig = field(default_factory=CaptureConfig) + pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) + discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) + scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig) + + @property + def parallel_tabs(self) -> int: + return self.celery.parallel_tabs + + @property + def fetch_concurrency(self) -> int: + return self.celery.fetch_concurrency + + @property + def block_resources(self) -> bool: + return self.celery.block_resources + +# Глобальный синглтон — используется по умолчанию во всех модулях. +settings = Settings() diff --git a/iaai_scraper/core/exceptions.py b/iaai_scraper/core/exceptions.py new file mode 100644 index 0000000..d3b2872 --- /dev/null +++ b/iaai_scraper/core/exceptions.py @@ -0,0 +1,14 @@ +class ScraperError(Exception): + """Базовое исключение скрапера.""" + + +class AntiBotDetectedError(ScraperError): + """Вызывается, когда сайт блокирует автоматизацию.""" + + +class SiteStructureChangedError(ScraperError): + """Вызывается, когда структура страницы изменилась и данных не хватает.""" + + +class ListingResumeError(ScraperError): + """Вызывается, когда resume по checkpoint больше недостижим.""" diff --git a/iaai_scraper/core/logs.py b/iaai_scraper/core/logs.py new file mode 100644 index 0000000..45e151a --- /dev/null +++ b/iaai_scraper/core/logs.py @@ -0,0 +1,55 @@ +import logging +import sys +from contextvars import ContextVar + +# Храним trace_id текущего потока/корутины. +TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") + + +class TraceIdFilter(logging.Filter): + # Добавляет trace_id в каждую запись лога для сквозной трассировки. + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = TRACE_ID.get() + return True + + +def set_trace_id(trace_id: str) -> None: + TRACE_ID.set(trace_id) + + +def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + # stderr — Docker и Celery prefork корректно его подхватывают. + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)] + if log_file: + handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + trace_filter = TraceIdFilter() + for handler in handlers: + handler.addFilter(trace_filter) + handler.setLevel(getattr(logging, level.upper(), logging.INFO)) + root = logging.getLogger() + root.setLevel(getattr(logging, level.upper(), logging.INFO)) + # Убираем старые хендлеры, чтобы не дублировать после fork. + for old_handler in list(root.handlers): + try: + old_handler.close() + except Exception: + pass + root.handlers.clear() + for handler in handlers: + root.addHandler(handler) + root.propagate = False + fmt = logging.Formatter( + "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s" + ) + for handler in root.handlers: + handler.setFormatter(fmt) + + for logger_name in ( + "celery.app.trace", + "celery.worker.request", + "celery.worker.strategy", + ): + noisy_logger = logging.getLogger(logger_name) + noisy_logger.handlers.clear() + noisy_logger.propagate = False + noisy_logger.disabled = True diff --git a/iaai_scraper/core/retry.py b/iaai_scraper/core/retry.py new file mode 100644 index 0000000..2fce32c --- /dev/null +++ b/iaai_scraper/core/retry.py @@ -0,0 +1,53 @@ +import logging +import random +import time +from collections.abc import Callable +from functools import wraps +from typing import Any + +from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError + +from .exceptions import AntiBotDetectedError + +logger = logging.getLogger("iaai_scraper.retry") + +# Типы исключений, при которых retry имеет смысл. +RETRYABLE_EXCEPTIONS = ( + PlaywrightTimeoutError, + Error, + ConnectionError, + OSError, + TimeoutError, + AntiBotDetectedError, +) + + +def retryable( + max_attempts: int, + delay_seconds: float = 2.5, + backoff_multiplier: float = 2.0, + jitter_seconds: float = 0.0, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + return func(*args, **kwargs) + except RETRYABLE_EXCEPTIONS as exc: + last_error = exc + logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) + if attempt < max_attempts: + sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1)) + if jitter_seconds > 0: + sleep_for += random.uniform(0, jitter_seconds) + logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for) + time.sleep(sleep_for) + if last_error is not None: + raise last_error + raise RuntimeError("Retry wrapper failed without a captured exception") + + return wrapper + + return decorator diff --git a/iaai_scraper/core/runtime_config.py b/iaai_scraper/core/runtime_config.py new file mode 100644 index 0000000..ded9f3d --- /dev/null +++ b/iaai_scraper/core/runtime_config.py @@ -0,0 +1,395 @@ +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +logger = logging.getLogger("iaai_scraper.runtime_config") + + +def _normalize_text(value: str) -> str: + return value.strip().casefold() + + +def _text_tuple(values: Any) -> tuple[str, ...]: + return tuple( + str(item).strip() + for item in (values or []) + if str(item).strip() + ) + + +def _int_tuple(values: Any) -> tuple[int, ...]: + result: list[int] = [] + for value in values or []: + try: + result.append(int(value)) + except (TypeError, ValueError): + continue + return tuple(result) + + +def _optional_int(value: Any) -> int | None: + if value in (None, ""): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _optional_bool(value: Any) -> bool | None: + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().casefold() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return None + + +@dataclass(slots=True) +class RuntimeSyncConfig: + name: str | None = None + ids_initial_size: int | None = None + ids_next_size: int | None = None + ids_max_pages: int | None = None + condition_check_enabled: bool | None = None + lane: str | None = None + only_new: bool | None = None + limit: int | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig": + data = data or {} + name = str(data.get("name")).strip() if data.get("name") else None + lane = str(data.get("lane")).strip() if data.get("lane") else None + return cls( + name=name or None, + ids_initial_size=_optional_int(data.get("ids_initial_size")), + ids_next_size=_optional_int(data.get("ids_next_size")), + ids_max_pages=_optional_int(data.get("ids_max_pages")), + condition_check_enabled=_optional_bool(data.get("condition_check_enabled")), + lane=lane or None, + only_new=_optional_bool(data.get("only_new")), + limit=_optional_int(data.get("limit")), + ) + + +@dataclass(slots=True) +class RuntimeListingConfig: + make: str | None = None + model: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig": + data = data or {} + make = str(data.get("make")).strip() if data.get("make") else None + model = str(data.get("model")).strip() if data.get("model") else None + return cls(make=make or None, model=model or None) + + +@dataclass(slots=True) +class RuntimeFieldFilters: + brands: tuple[str, ...] = () + models: tuple[str, ...] = () + years: tuple[int, ...] = () + body_types: tuple[str, ...] = () + colors: tuple[str, ...] = () + drives: tuple[str, ...] = () + gearboxes: tuple[str, ...] = () + locations: tuple[str, ...] = () + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters": + data = data or {} + return cls( + brands=_text_tuple(data.get("brands")), + models=_text_tuple(data.get("models")), + years=_int_tuple(data.get("years")), + body_types=_text_tuple(data.get("body_types")), + colors=_text_tuple(data.get("colors")), + drives=_text_tuple(data.get("drives")), + gearboxes=_text_tuple(data.get("gearboxes")), + locations=_text_tuple(data.get("locations")), + ) + + def is_empty(self) -> bool: + return not any([ + self.brands, + self.models, + self.years, + self.body_types, + self.colors, + self.drives, + self.gearboxes, + self.locations, + ]) + + def matches(self, values: dict[str, Any]) -> bool: + return all([ + self._match_text(self.brands, values.get("brand")), + self._match_text(self.models, values.get("model")), + self._match_int(self.years, values.get("year")), + self._match_text(self.body_types, values.get("body_type")), + self._match_text(self.colors, values.get("color")), + self._match_text(self.drives, values.get("drive")), + self._match_text(self.gearboxes, values.get("gearbox")), + self._match_text(self.locations, values.get("location")), + ]) + + @staticmethod + def _match_text(allowed: tuple[str, ...], value: Any) -> bool: + if not allowed: + return True + normalized = _normalize_text(str(value or "")) + return normalized in {_normalize_text(item) for item in allowed} + + @staticmethod + def _match_int(allowed: tuple[int, ...], value: Any) -> bool: + if not allowed: + return True + parsed = _optional_int(value) + return parsed in set(allowed) + + +@dataclass(slots=True) +class RuntimeRangeFilter: + min: int | None = None + max: int | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter": + data = data or {} + return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max"))) + + def is_empty(self) -> bool: + return self.min is None and self.max is None + + def matches(self, value: Any) -> bool: + parsed = _optional_int(value) + if parsed is None: + return self.is_empty() + if self.min is not None and parsed < self.min: + return False + if self.max is not None and parsed > self.max: + return False + return True + + +@dataclass(slots=True) +class RuntimeFlagFilters: + damaged_only: bool | None = None + run_and_drive: bool | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters": + data = data or {} + return cls( + damaged_only=_optional_bool(data.get("damaged_only")), + run_and_drive=_optional_bool(data.get("run_and_drive")), + ) + + def is_empty(self) -> bool: + return self.damaged_only is None and self.run_and_drive is None + + def matches(self, values: dict[str, Any]) -> bool: + if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only: + return False + if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive: + return False + return True + + +@dataclass(slots=True) +class RuntimeFiltersConfig: + include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) + exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) + price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) + mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) + flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters) + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig": + data = data or {} + legacy_fields = RuntimeFieldFilters.from_dict(data) + include_payload = data.get("include") + exclude_payload = data.get("exclude") + + flat_exclude_payload = { + "brands": data.get("exclude_brands"), + "models": data.get("exclude_models"), + "years": data.get("exclude_years"), + "body_types": data.get("exclude_body_types"), + "colors": data.get("exclude_colors"), + "drives": data.get("exclude_drives"), + "gearboxes": data.get("exclude_gearboxes"), + "locations": data.get("exclude_locations"), + } + + include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields + exclude = ( + RuntimeFieldFilters.from_dict(exclude_payload) + if exclude_payload is not None + else RuntimeFieldFilters.from_dict(flat_exclude_payload) + ) + + return cls( + include=include, + exclude=exclude, + price=RuntimeRangeFilter.from_dict(data.get("price")), + mileage=RuntimeRangeFilter.from_dict(data.get("mileage")), + flags=RuntimeFlagFilters.from_dict(data.get("flags")), + ) + + def is_empty(self) -> bool: + return all([ + self.include.is_empty(), + self.exclude.is_empty(), + self.price.is_empty(), + self.mileage.is_empty(), + self.flags.is_empty(), + ]) + + def matches(self, values: dict[str, Any]) -> bool: + if not self.include.matches(values): + return False + if not self._matches_exclude(values): + return False + if not self.price.matches(values.get("price")): + return False + if not self.mileage.matches(values.get("mileage")): + return False + if not self.flags.matches(values): + return False + return True + + def _matches_exclude(self, values: dict[str, Any]) -> bool: + if self.exclude.is_empty(): + return True + return not self.exclude.matches(values) + + +@dataclass(slots=True) +class RuntimeMobileDeSegment: + make: str | None = None + make_id: str | None = None + model: str | None = None + model_id: str | None = None + search_url: str | None = None + only_new: bool | None = None + price_min: str | None = None + price_max: str | None = None + year_min: str | None = None + year_max: str | None = None + start_page: int = 1 + max_pages: int | None = None + label: str | None = None + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeSegment | None": + data = data or {} + make = str(data.get("make") or "").strip() or None + make_id = str(data.get("make_id") or data.get("makeId") or "").strip() or None + model = str(data.get("model") or "").strip() or None + model_id = str(data.get("model_id") or data.get("modelId") or "").strip() or None + search_url = str(data.get("search_url") or data.get("searchUrl") or data.get("listing_url") or "").strip() or None + if not make_id and not make and not search_url: + return None + label = str(data.get("label") or "").strip() or None + return cls( + make=make, + make_id=make_id, + model=model, + model_id=model_id, + search_url=search_url, + only_new=_optional_bool(data.get("only_new")), + price_min=str(data.get("price_min") or "").strip() or None, + price_max=str(data.get("price_max") or "").strip() or None, + year_min=str(data.get("year_min") or "").strip() or None, + year_max=str(data.get("year_max") or "").strip() or None, + start_page=max(1, _optional_int(data.get("start_page")) or 1), + max_pages=_optional_int(data.get("max_pages")), + label=label, + ) + + def to_task_kwargs(self) -> dict[str, Any]: + return { + "make": self.make, + "make_id": self.make_id, + "model": self.model, + "model_id": self.model_id, + "search_url": self.search_url, + "only_new": self.only_new, + "price_min": self.price_min, + "price_max": self.price_max, + "year_min": self.year_min, + "year_max": self.year_max, + "start_page": self.start_page, + "max_pages": self.max_pages, + "label": self.label or self.display_name, + } + + @property + def display_name(self) -> str: + if self.label: + return self.label + if self.search_url: + return "filtered-url" + parts = [part for part in [self.make, self.model] if part] + return " / ".join(parts) or self.make_id or "mobilede-segment" + + +@dataclass(slots=True) +class RuntimeMobileDeConfig: + segments: tuple[RuntimeMobileDeSegment, ...] = () + + @classmethod + def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeMobileDeConfig": + data = data or {} + raw_segments = data.get("segments") + segments: list[RuntimeMobileDeSegment] = [] + if isinstance(raw_segments, list): + for item in raw_segments: + if not isinstance(item, dict): + continue + segment = RuntimeMobileDeSegment.from_dict(item) + if segment is not None: + segments.append(segment) + return cls(segments=tuple(segments)) + + def is_empty(self) -> bool: + return not self.segments + + +@dataclass(slots=True) +class RuntimeConfig: + sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig) + listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig) + filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig) + mobilede: RuntimeMobileDeConfig = field(default_factory=RuntimeMobileDeConfig) + + @classmethod + def from_file(cls, config_path: str | None) -> "RuntimeConfig": + if not config_path: + return cls() + path = Path(config_path) + if not path.exists(): + logger.info("Runtime config file not found: %s", path) + return cls() + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + logger.warning("Failed to read runtime config %s: %s", path, exc) + return cls() + return cls( + sync=RuntimeSyncConfig.from_dict(payload.get("sync")), + listing=RuntimeListingConfig.from_dict(payload.get("listing")), + filters=RuntimeFiltersConfig.from_dict(payload.get("filters")), + mobilede=RuntimeMobileDeConfig.from_dict(payload.get("mobilede")), + ) diff --git a/iaai_scraper/core/utils.py b/iaai_scraper/core/utils.py new file mode 100644 index 0000000..8c8cad5 --- /dev/null +++ b/iaai_scraper/core/utils.py @@ -0,0 +1,72 @@ +import json +import re +from pathlib import Path +from typing import Any, Callable, Iterable + + +def save_to_json(data: Any, filename: str | Path) -> None: + path = Path(filename) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def first_non_empty(values: Iterable[Any]) -> Any | None: + for value in values: + if value not in (None, "", [], {}, ()): + return value + return None + + +# Регулярные выражения для VIN, lot и price. +VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) +LOT_RE = re.compile(r"\b(\d{7,10})\b") +PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") + + +def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: + # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве. + found = [] + if _depth >= max_depth: + return found + if isinstance(obj, dict): + for key, value in obj.items(): + if key.lower() in target_keys: + found.append(value) + found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) + elif isinstance(obj, list): + for item in obj: + found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) + return found + + +def deep_find_all_keys( + payloads: list, + field_map: dict[str, set[str]], + max_depth: int = 64, +) -> dict[str, list]: + """Извлекает все нужные поля за один проход по JSON.""" + # Готовим обратную карту: нормализованный ключ -> имя поля. + reverse: dict[str, str] = {} + for field_name, keys in field_map.items(): + for k in keys: + reverse[k.lower()] = field_name + + result: dict[str, list] = {f: [] for f in field_map} + + def _recurse(obj: Any, depth: int) -> None: + if depth >= max_depth: + return + if isinstance(obj, dict): + for k, v in obj.items(): + field = reverse.get(k.lower()) + if field is not None: + result[field].append(v) + _recurse(v, depth + 1) + elif isinstance(obj, list): + for item in obj: + _recurse(item, depth + 1) + + for payload in payloads: + _recurse(payload, 0) + + return result diff --git a/iaai_scraper/discovery/__init__.py b/iaai_scraper/discovery/__init__.py new file mode 100644 index 0000000..86402b2 --- /dev/null +++ b/iaai_scraper/discovery/__init__.py @@ -0,0 +1,15 @@ +from .sitemap import ( + SitemapDiscoveryError, + SitemapDiscoveryResult, + SitemapDiscoveryStats, + discover_vehicle_urls_from_sitemap, + discover_vehicle_urls_from_sitemap_with_stats, +) + +__all__ = [ + "SitemapDiscoveryError", + "SitemapDiscoveryResult", + "SitemapDiscoveryStats", + "discover_vehicle_urls_from_sitemap", + "discover_vehicle_urls_from_sitemap_with_stats", +] diff --git a/iaai_scraper/discovery/sitemap.py b/iaai_scraper/discovery/sitemap.py new file mode 100644 index 0000000..dc50104 --- /dev/null +++ b/iaai_scraper/discovery/sitemap.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import gzip +import io +import logging +import re +from dataclasses import dataclass, field +from typing import Iterable +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen, ProxyHandler, build_opener +import xml.etree.ElementTree as ET + +logger = logging.getLogger("iaai_scraper.discovery.sitemap") + +DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" +_SITEMAP_TIMEOUT_SECONDS = 30 +_LOC_TAG_RE = re.compile(rb"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) + + +class SitemapDiscoveryError(RuntimeError): + pass + + +def _is_vehicle_sitemap_url(url: str) -> bool: + lowered = url.strip().lower() + # Берём только sitemap с авто. + if "sitemapbranches" in lowered or "sitemapauctions" in lowered: + return False + return lowered.endswith(".xml") or lowered.endswith(".xml.gz") + + +def _normalize_vehicle_url(url: str) -> str: + parts = urlsplit(url.strip()) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + + +def _download_bytes(url: str, proxy_url: str | None = None) -> bytes: + request = Request( + url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36" + ), + "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8", + "Accept-Encoding": "gzip", + }, + ) + if proxy_url: + handler = ProxyHandler({"http": proxy_url, "https": proxy_url}) + opener = build_opener(handler) + response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + else: + response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + with response: + payload = response.read() + encoding = str(response.headers.get("Content-Encoding") or "").lower() + if encoding == "gzip" or url.lower().endswith(".gz"): + return gzip.GzipFile(fileobj=io.BytesIO(payload)).read() + return payload + + +def _local_name(tag: str) -> str: + if "}" in tag: + return tag.rsplit("}", 1)[1] + return tag + + +def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]: + for match in _LOC_TAG_RE.finditer(xml_bytes): + try: + value = match.group(1).decode("utf-8", errors="ignore").strip() + except Exception: + continue + if value: + yield value + + +def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]: + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError as exc: + fallback_values = list(_iter_loc_values_fallback(xml_bytes)) + if fallback_values: + logger.warning( + "Falling back to regex sitemap loc extraction after XML parse error: %s", + exc, + ) + yield from fallback_values + return + raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc + + for element in root.iter(): + if _local_name(element.tag) != "loc": + continue + if not element.text: + continue + value = element.text.strip() + if value: + yield value + + +def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]: + result: list[str] = [] + seen: set[str] = set() + for url in urls: + normalized = _normalize_vehicle_url(url) + if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized: + continue + if normalized in seen: + continue + seen.add(normalized) + result.append(normalized) + return result + + +def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool: + return any("/vehicledetail/" in url.lower() for url in urls) + + +def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]: + logger.info("Downloading sitemap index: %s", index_url) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) + return deduped + + +@dataclass +class SitemapDiscoveryStats: + transport: str = "http" + fetched_sitemaps: int = 0 + blocked_sitemaps: int = 0 + malformed_sitemaps: int = 0 + direct_probe_hits: int = 0 + direct_probe_misses: int = 0 + + +@dataclass +class SitemapDiscoveryResult: + vehicle_urls: list[str] = field(default_factory=list) + stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats) + + +def discover_vehicle_urls_from_sitemap_with_stats( + settings=None, + index_url: str = DEFAULT_SITEMAP_INDEX_URL, +) -> SitemapDiscoveryResult: + """Возвращает URL и статистику.""" + proxy_url = None + if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server: + proxy_url = settings.proxy.server + stats = SitemapDiscoveryStats(transport="http") + logger.info("Downloading sitemap index: %s", index_url) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + stats.fetched_sitemaps += 1 + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + stats.malformed_sitemaps += 1 + continue + except Exception as exc: + logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc) + stats.blocked_sitemaps += 1 + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) + return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats) diff --git a/iaai_scraper/fast_sync.py b/iaai_scraper/fast_sync.py new file mode 100644 index 0000000..92911ce --- /dev/null +++ b/iaai_scraper/fast_sync.py @@ -0,0 +1,472 @@ +from __future__ import annotations + +import concurrent.futures +import logging +import random +import time +from dataclasses import dataclass +from typing import Any, Callable + +from .browser.fast_client import FastListingVehicle, IAAIFastClient +from .core.runtime_config import RuntimeConfig +from .parsing.fast_mapper import INACTIVE_STATUS_VALUES, FastCarMapper +from .storage.db import PersistenceService +from .storage.schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.fast_sync") + + +@dataclass(slots=True) +class FastSyncStats: + ids_fetched: int = 0 + cars_upserted: int = 0 + cars_failed: int = 0 + cars_filtered: int = 0 + images_upserted: int = 0 + skipped_existing: int = 0 + protection_events: int = 0 + + +def passes_condition_check(row: FastListingVehicle) -> bool: + if row.timed_auction_closed: + return False + status = (row.inventory_status or "").strip().upper() + return status not in INACTIVE_STATUS_VALUES + + +class FastSyncEngine: + """Full iaai-fast style sync pipeline adapted to this project's storage. + + Flow: hidden listing payloads -> concurrent ProductDetailsVM HTTP fetch -> + CarRecord preparation in memory -> single batch DB upsert. Browser is used + only inside IAAIFastClient to refresh cookies when IAAI challenge appears. + """ + + def __init__( + self, + *, + client: IAAIFastClient, + mapper: FastCarMapper, + persistence: PersistenceService, + batch_size: int, + fetch_concurrency: int, + report_progress: Callable[[str, Any], None] | None = None, + ) -> None: + self.client = client + self.mapper = mapper + self.persistence = persistence + self.batch_size = max(1, int(batch_size)) + self.fetch_concurrency = max(1, int(fetch_concurrency)) + self.report_progress = report_progress + + @staticmethod + def _is_transient_detail_error(exc: Exception) -> bool: + text = str(exc).lower() + return any( + marker in text + for marker in ( + "sslerror", + "ssleoferror", + "unexpected_eof_while_reading", + "eof occurred in violation of protocol", + "max retries exceeded", + "read timed out", + "readtimeout", + "connection reset", + "connection aborted", + "connection closed", + "temporarily unavailable", + "too many requests", + "status=429", + "status=500", + "status=502", + "status=503", + "status=504", + ) + ) + + def sync_listing( + self, + *, + runtime_config: RuntimeConfig, + make: str | None = None, + model: str | None = None, + lane: str = "iaai_cars", + limit: int | None = None, + only_new: bool = False, + listing_url: str | None = None, + max_pages: int | None = None, + skip_mark_sold: bool = False, + ) -> dict[str, Any]: + del lane + started_at = time.perf_counter() + stats = FastSyncStats() + errors: list[dict[str, str]] = [] + selected: dict[str, FastListingVehicle] = {} + rows_seen = 0 + rows_skipped_condition = 0 + + filters = runtime_config.filters + logger.info( + "Fast HTTP-first listing started: make=%s listing_url=%s max_pages=%s concurrency=%s batch_size=%s", + make or "ALL", + listing_url or "default", + max_pages, + self.fetch_concurrency, + self.batch_size, + ) + for vehicle in self.client.iter_listing_vehicles( + listing_start_url=listing_url, + make=make, + max_pages=max_pages, + ): + rows_seen += 1 + if runtime_config.sync.condition_check_enabled and not passes_condition_check(vehicle): + rows_skipped_condition += 1 + continue + if vehicle.inventory_id in selected: + continue + selected[vehicle.inventory_id] = vehicle + if limit is not None and limit > 0 and len(selected) >= limit: + break + + candidates = list(selected.values()) + all_listing_origin_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates} + if only_new and candidates: + origin_urls = [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates] + origin_ids = [f"iaai:{v.inventory_id}" for v in candidates] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(origin_urls, origin_ids) + fresh: list[FastListingVehicle] = [] + for vehicle in candidates: + if f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}" in existing_urls or f"iaai:{vehicle.inventory_id}" in existing_ids: + stats.skipped_existing += 1 + continue + fresh.append(vehicle) + candidates = fresh + + self._progress( + "fast_listing_collected", + rows_seen=rows_seen, + rows_filtered_condition=rows_skipped_condition, + rows_selected=len(candidates), + skipped_existing=stats.skipped_existing, + ) + logger.info( + "Fast HTTP-first listing collected: rows_seen=%d selected=%d skipped_existing=%d filtered_condition=%d only_new=%s", + rows_seen, + len(candidates), + stats.skipped_existing, + rows_skipped_condition, + only_new, + ) + + scan_completed = not (limit is not None and limit > 0) and not errors + + if not candidates: + return self._result( + started_at=started_at, + stats=stats, + failures=errors, + listing={ + "mode": "fast_hidden_payload", + "vehicles_collected": 0, + "vehicle_urls": [], + "early_stopped": False, + "truncated_by_time_budget": False, + "rows_seen": rows_seen, + "rows_filtered_condition": rows_skipped_condition, + }, + all_listing_origin_urls=all_listing_origin_urls, + full_scan_completed=scan_completed, + ) + + prepared_rows: list[CarRecord] = [] + db_processed = 0 + retry_candidates: list[FastListingVehicle] = [] + started_details = time.perf_counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor: + future_to_vehicle = { + executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle + for vehicle in candidates + } + for index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1): + vehicle = future_to_vehicle[future] + try: + payload = future.result() + record = self.mapper.map_payload_to_record( + detail_payload=payload, + vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}", + listing_vehicle=vehicle, + ) + if model and model.casefold() not in record.model.casefold(): + stats.cars_filtered += 1 + continue + if not filters.matches({ + "brand": record.brand, + "model": record.model, + "year": record.year, + "body_type": record.body_type, + "color": record.color, + "drive": record.drive, + "gearbox": record.gearbox, + "price": record.price, + "mileage": record.mileage, + }): + stats.cars_filtered += 1 + continue + prepared_rows.append(record) + stats.ids_fetched += 1 + if len(prepared_rows) >= self.batch_size: + db_processed += self._flush_db_records( + rows=prepared_rows, + stats=stats, + errors=errors, + processed=db_processed + len(prepared_rows), + total=len(candidates), + ) + prepared_rows.clear() + except Exception as exc: + stats.cars_failed += 1 + errors.append({"vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}", "error": str(exc)}) + if _looks_like_protection(exc): + stats.protection_events += 1 + if self._is_transient_detail_error(exc): + retry_candidates.append(vehicle) + logger.info("Fast detail transient failure queued for retry inventory_id=%s: %s", vehicle.inventory_id, exc) + else: + logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc) + + if index % 100 == 0 or index == len(candidates): + elapsed = max(0.001, time.perf_counter() - started_details) + if self.client._settings.scraping_profile.verbose_progress_logs: + logger.info( + "Fast HTTP-first details progress: processed=%d/%d ok=%d failed=%d queued_db=%d rate=%.2f/s", + index, + len(candidates), + stats.ids_fetched, + stats.cars_failed, + len(prepared_rows), + index / elapsed, + ) + self._progress( + "fast_detail_progress", + processed=index, + total=len(candidates), + ids_fetched=stats.ids_fetched, + cars_failed=stats.cars_failed, + queued_for_db=len(prepared_rows), + throughput=round(index / elapsed, 2), + ) + + if retry_candidates: + retry_started = time.perf_counter() + retry_workers = max(1, min(8, self.fetch_concurrency // 2, len(retry_candidates))) + retry_errors: list[dict[str, str]] = [] + logger.info( + "Fast HTTP-first retrying transient detail failures: total=%d workers=%d", + len(retry_candidates), + retry_workers, + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=retry_workers) as executor: + future_to_vehicle = { + executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle + for vehicle in retry_candidates + } + for retry_index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1): + vehicle = future_to_vehicle[future] + try: + payload = future.result() + record = self.mapper.map_payload_to_record( + detail_payload=payload, + vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}", + listing_vehicle=vehicle, + ) + if model and model.casefold() not in record.model.casefold(): + stats.cars_filtered += 1 + continue + if not filters.matches({ + "brand": record.brand, + "model": record.model, + "year": record.year, + "body_type": record.body_type, + "color": record.color, + "drive": record.drive, + "gearbox": record.gearbox, + "price": record.price, + "mileage": record.mileage, + }): + stats.cars_filtered += 1 + continue + prepared_rows.append(record) + stats.ids_fetched += 1 + stats.cars_failed = max(0, stats.cars_failed - 1) + if len(prepared_rows) >= self.batch_size: + db_processed += self._flush_db_records( + rows=prepared_rows, + stats=stats, + errors=errors, + processed=db_processed + len(prepared_rows), + total=len(candidates), + ) + prepared_rows.clear() + except Exception as exc: + retry_errors.append({ + "vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}", + "error": str(exc), + }) + if _looks_like_protection(exc): + stats.protection_events += 1 + if retry_index % 100 == 0 or retry_index == len(retry_candidates): + elapsed = max(0.001, time.perf_counter() - retry_started) + logger.info( + "Fast HTTP-first retry progress: processed=%d/%d recovered=%d remaining_failed=%d rate=%.2f/s", + retry_index, + len(retry_candidates), + len(retry_candidates) - len(retry_errors), + len(retry_errors), + retry_index / elapsed, + ) + transient_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in retry_candidates} + errors = [error for error in errors if error.get("vehicle_url") not in transient_urls] + errors.extend(retry_errors) + + if prepared_rows: + db_processed += self._flush_db_records( + rows=prepared_rows, + stats=stats, + errors=errors, + processed=db_processed + len(prepared_rows), + total=len(candidates), + ) + prepared_rows.clear() + + self.client.persist_session_state() + + scan_completed = not (limit is not None and limit > 0) and not errors + mark_sold_scope_partial = bool( + (limit is not None and limit > 0) + or make + or model + or listing_url + or only_new + ) + if all_listing_origin_urls and not mark_sold_scope_partial and not skip_mark_sold and scan_completed: + try: + sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls, lane="iaai") + except Exception as exc: + sold_count = 0 + logger.warning("Fast sold reconcile failed: %s", exc) + else: + sold_count = 0 + + listing = { + "mode": "fast_hidden_payload", + "vehicles_collected": len(candidates), + "vehicle_urls": [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates], + "early_stopped": False, + "truncated_by_time_budget": False, + "rows_seen": rows_seen, + "rows_filtered_condition": rows_skipped_condition, + "sold_marked": sold_count, + } + return self._result( + started_at=started_at, + stats=stats, + failures=errors, + listing=listing, + all_listing_origin_urls=all_listing_origin_urls, + full_scan_completed=scan_completed, + ) + + def _result( + self, + *, + started_at: float, + stats: FastSyncStats, + failures: list[dict[str, str]], + listing: dict[str, Any], + all_listing_origin_urls: set[str], + full_scan_completed: bool, + ) -> dict[str, Any]: + status = "success" if not failures else ("partial_success" if stats.cars_upserted else "failed") + total = int(listing.get("vehicles_collected") or 0) + fail_ratio = (stats.cars_failed / total) if total > 0 else 0.0 + protection_ratio = (stats.protection_events / total) if total > 0 else 0.0 + anti_bot_detected = total > 0 and ((stats.protection_events >= 30 and protection_ratio >= 0.10) or fail_ratio >= 0.30) + return { + "status": status, + "listing": listing, + "total": total, + "total_discovered": total, + "skipped_existing": stats.skipped_existing, + "cars_upserted": stats.cars_upserted, + "cars_failed": stats.cars_failed, + "cars_filtered": stats.cars_filtered, + "images_upserted": stats.images_upserted, + "protection_events": stats.protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + "full_scan_completed": full_scan_completed and not anti_bot_detected, + "anti_bot_detected": anti_bot_detected, + "fail_ratio": round(fail_ratio, 4), + "protection_ratio": round(protection_ratio, 4), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + } + + def _progress(self, stage: str, **meta: Any) -> None: + if self.report_progress is None: + return + try: + self.report_progress(stage, **meta) + except Exception: + pass + + def _fetch_detail_payload(self, inventory_id: str) -> dict[str, Any]: + jitter = float(self.client._settings.scraping_profile.request_jitter_max_s) + if jitter > 0: + time.sleep(random.uniform(0.0, jitter)) + return self.client.fetch_vehicle_detail_payload(inventory_id) + + def _flush_db_records( + self, + *, + rows: list[CarRecord], + stats: FastSyncStats, + errors: list[dict[str, str]], + processed: int, + total: int, + ) -> int: + if not rows: + return 0 + batch = list(rows) + try: + upsert = self.persistence.upsert_cars_batch(batch) + stats.cars_upserted += int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0)) + stats.images_upserted += int(upsert.get("images_upserted", 0)) + except Exception as exc: + stats.cars_failed += len(batch) + errors.append({"vehicle_url": f"db_batch_{processed - len(batch)}", "error": str(exc)}) + logger.exception("Fast DB apply failed processed=%s size=%s: %s", processed, len(batch), exc) + self._progress( + "fast_db_progress", + processed=processed, + total=total, + cars_upserted=stats.cars_upserted, + cars_failed=stats.cars_failed, + images_upserted=stats.images_upserted, + ) + logger.info( + "Fast HTTP-first DB batch: processed=%d/%d batch=%d upserted=%d failed=%d images=%d", + processed, + total, + len(batch), + stats.cars_upserted, + stats.cars_failed, + stats.images_upserted, + ) + return len(batch) + + +def _looks_like_protection(exc: Exception) -> bool: + message = str(exc).lower() + return any(token in message for token in ("captcha", "antibot", "challenge", "blocked", "403", "429", "incapsula")) diff --git a/iaai_scraper/mobile_de/__init__.py b/iaai_scraper/mobile_de/__init__.py new file mode 100644 index 0000000..51e4f1c --- /dev/null +++ b/iaai_scraper/mobile_de/__init__.py @@ -0,0 +1,3 @@ +from .client import MobileDeClient +from .scraper import MobileDeScraper + diff --git a/iaai_scraper/mobile_de/client.py b/iaai_scraper/mobile_de/client.py new file mode 100644 index 0000000..a02a42d --- /dev/null +++ b/iaai_scraper/mobile_de/client.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import logging +import threading +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from collections.abc import Callable, Iterable +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import requests + +from .flight import extract_detail_listing, extract_search_results +from .models import MobileDeListing, MobileDeSearchPage + +logger = logging.getLogger("mobile_de.client") + +BASE_URL = "https://www.mobile.de" +SEARCH_PATH = "/ru/транспортные-средства/поиск.html" +DETAIL_PATH = "/ru/транспортные-средства/подробности.html" +DEFAULT_HEADERS = { + "user-agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" + ), + "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "accept-language": "ru,en;q=0.9,de;q=0.8", +} + + +class MobileDeClient: + """HTTP client for mobile.de search/detail pages.""" + + def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None: + self.session = session or requests.Session() + self.session.headers.update(DEFAULT_HEADERS) + self.delay_seconds = max(0.0, delay_seconds) + + @classmethod + def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient": + session = requests.Session() + adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0) + session.mount("https://", adapter) + session.mount("http://", adapter) + return cls(session=session, delay_seconds=delay_seconds) + + @staticmethod + def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str: + make = str(make_id).strip() + model = str(model_id).strip() if model_id is not None else "" + return f"{make};{model};;" + + @staticmethod + def build_search_url(page_number: int = 1, **params: str | int | None) -> str: + query = { + "sb": "rel", + "od": "up", + "vc": "Car", + "s": "Car", + "pageNumber": page_number, + } + query.update({key: value for key, value in params.items() if value is not None}) + return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}" + + @staticmethod + def build_search_url_from_existing( + search_url: str, + *, + page_number: int | None = None, + **params: str | int | None, + ) -> str: + parts = urlsplit(search_url) + query_items = [ + (key, value) + for key, value in parse_qsl(parts.query, keep_blank_values=True) + if key != "pageNumber" and key not in params + ] + if page_number is not None: + query_items.append(("pageNumber", str(page_number))) + query_items.extend((key, str(value)) for key, value in params.items() if value is not None) + scheme = parts.scheme or "https" + netloc = parts.netloc or urlsplit(BASE_URL).netloc + path = parts.path or SEARCH_PATH + return urlunsplit((scheme, netloc, path, urlencode(query_items), "")) + + @staticmethod + def build_detail_url(listing_id: str | int) -> str: + query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"}) + return f"{BASE_URL}{DETAIL_PATH}?{query}" + + def fetch_html(self, url: str, *, timeout: int = 30) -> str: + response = self.session.get(url, timeout=timeout) + response.raise_for_status() + return response.text + + def fetch_search_page( + self, + page_number: int = 1, + *, + search_url: str | None = None, + **params: str | int | None, + ) -> MobileDeSearchPage: + url = ( + self.build_search_url_from_existing(search_url, page_number=page_number, **params) + if search_url + else self.build_search_url(page_number=page_number, **params) + ) + html = self.fetch_html(url) + raw = extract_search_results(html) + listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)] + return MobileDeSearchPage( + url=url, + page_number=page_number, + total_results=raw.get("numResultsTotal"), + listings=listings, + raw_search_results=raw, + ) + + def iter_search_pages( + self, + *, + start_page: int = 1, + max_pages: int | None = None, + search_url: str | None = None, + stop_after_empty: bool = True, + progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None, + **params: str | int | None, + ) -> Iterable[MobileDeSearchPage]: + page_number = start_page + pages_seen = 0 + logger.debug( + "mobile.de search window started: start_page=%s max_pages=%s params=%s", + start_page, + max_pages, + {key: value for key, value in params.items() if value is not None}, + ) + while max_pages is None or pages_seen < max_pages: + logger.debug("mobile.de fetching search page=%s", page_number) + page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params) + page_meta = { + "page_number": page.page_number, + "pages_seen": pages_seen + 1, + "max_pages": max_pages, + "listing_count": len(page.listings), + "total_results": page.total_results, + } + logger.debug( + "mobile.de fetched search page=%s listings=%s total_results=%s", + page.page_number, + len(page.listings), + page.total_results, + ) + if progress_callback is not None: + progress_callback(page, page_meta) + if stop_after_empty and not page.listings: + logger.debug("mobile.de stopping search window on empty page=%s", page.page_number) + break + yield page + pages_seen += 1 + page_number += 1 + if self.delay_seconds: + time.sleep(self.delay_seconds) + logger.debug( + "mobile.de search window finished: pages_seen=%s next_page=%s", + pages_seen, + page_number, + ) + + def fetch_search_pages_concurrent( + self, + *, + start_page: int = 1, + max_pages: int = 1, + workers: int = 8, + search_url: str | None = None, + stop_after_empty: bool = True, + progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None, + **params: str | int | None, + ) -> list[MobileDeSearchPage]: + max_pages = max(1, int(max_pages)) + workers = max(1, min(int(workers), max_pages)) + page_numbers = list(range(max(1, int(start_page)), max(1, int(start_page)) + max_pages)) + pages_by_number: dict[int, MobileDeSearchPage] = {} + thread_local = threading.local() + + def _fetch_page(page_number: int) -> MobileDeSearchPage: + client = getattr(thread_local, "client", None) + if client is None: + client = MobileDeClient.for_worker(delay_seconds=0) + thread_local.client = client + return client.fetch_search_page(page_number=page_number, search_url=search_url, **params) + + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = { + executor.submit(_fetch_page, page_number): page_number + for page_number in page_numbers + } + for future in as_completed(futures): + page_number = futures[future] + page = future.result() + pages_by_number[page_number] = page + if progress_callback is not None: + progress_callback( + page, + { + "page_number": page.page_number, + "pages_seen": len(pages_by_number), + "max_pages": max_pages, + "listing_count": len(page.listings), + "total_results": page.total_results, + }, + ) + ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number] + if stop_after_empty: + non_empty_pages: list[MobileDeSearchPage] = [] + for page in ordered_pages: + if not page.listings: + break + non_empty_pages.append(page) + return non_empty_pages + return ordered_pages + + def fetch_detail(self, listing_id: str | int) -> dict: + html = self.fetch_html(self.build_detail_url(listing_id)) + return extract_detail_listing(html) + + @staticmethod + def _map_listing(item: dict) -> MobileDeListing: + listing_id = str(item.get("id") or item.get("adId") or "") + attr = item.get("attr") if isinstance(item.get("attr"), dict) else {} + contact = item.get("contact") if isinstance(item.get("contact"), dict) else {} + location = ", ".join( + part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part + ) or None + return MobileDeListing( + id=listing_id, + url=MobileDeClient.build_detail_url(listing_id) if listing_id else "", + title=item.get("shortTitle"), + subtitle=item.get("subTitle"), + price=item.get("p"), + seller_name=contact.get("name"), + seller_type=contact.get("type") or item.get("st"), + location=location, + first_registration=attr.get("fr"), + mileage=attr.get("ml"), + power=attr.get("pw"), + fuel=attr.get("ft"), + transmission=attr.get("tr"), + raw=item, + ) diff --git a/iaai_scraper/mobile_de/flight.py b/iaai_scraper/mobile_de/flight.py new file mode 100644 index 0000000..b85162f --- /dev/null +++ b/iaai_scraper/mobile_de/flight.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import json +import re +from typing import Any + +NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL) + + +def extract_next_flight_strings(html: str) -> list[str]: + """Extract decoded Next.js Flight chunks from mobile.de HTML.""" + chunks: list[str] = [] + for match in NEXT_FLIGHT_RE.finditer(html): + raw = match.group(1) + try: + chunks.append(json.loads(f'"{raw}"')) + except json.JSONDecodeError: + # Fallback keeps parser useful if one chunk has non-standard escaping. + chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore")) + return chunks + + +def extract_json_object_after(text: str, marker: str) -> dict[str, Any] | None: + """Return JSON object that starts immediately after a marker in a decoded Flight chunk.""" + marker_index = text.find(marker) + if marker_index < 0: + return None + start = text.find("{", marker_index + len(marker)) + if start < 0: + return None + + depth = 0 + in_string = False + escaped = False + for index in range(start, len(text)): + char = text[index] + if in_string: + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + in_string = False + continue + if char == '"': + in_string = True + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + if depth == 0: + candidate = text[start : index + 1] + try: + return json.loads(candidate) + except json.JSONDecodeError: + return None + return None + + +def extract_search_results(html: str) -> dict[str, Any]: + """Extract searchResults from mobile.de SRP HTML.""" + for chunk in extract_next_flight_strings(html): + if '"eventScope":"page-srp"' not in chunk or '"searchResults"' not in chunk: + continue + results = extract_json_object_after(chunk, '"searchResults":') + if isinstance(results, dict): + return results + return {} + + +def extract_detail_listing(html: str) -> dict[str, Any]: + """Extract listing object from mobile.de VIP/detail HTML.""" + for chunk in extract_next_flight_strings(html): + if '"eventScope":"page-vip"' not in chunk or '"listing"' not in chunk: + continue + listing = extract_json_object_after(chunk, '"listing":') + if isinstance(listing, dict): + return listing + return {} diff --git a/iaai_scraper/mobile_de/mapper.py b/iaai_scraper/mobile_de/mapper.py new file mode 100644 index 0000000..7bb9b9c --- /dev/null +++ b/iaai_scraper/mobile_de/mapper.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +import hashlib +import re +from datetime import datetime, timezone +from typing import Any + +from ..storage.schemas import CarRecord, ImageRecord +from .client import MobileDeClient +from .models import MobileDeListing + +_BODY_MAP = { + "cabrio": "OPEN", + "кабриолет": "OPEN", + "limousine": "SEDAN", + "седан": "SEDAN", + "suv": "SUV", + "внедорожник": "SUV", + "kombi": "STATION_WAGON", + "универсал": "STATION_WAGON", + "van": "MINIVAN", + "фургон": "MINIVAN", + "coupe": "COUPE", + "купе": "COUPE", + "kleinwagen": "HATCHBACK", + "маленький": "HATCHBACK", +} + +_GEARBOX_MAP = { + "автомат": "AT", + "automatic": "AT", + "механ": "MT", + "manual": "MT", + "cvt": "CVT", +} + +_COLOR_MAP = { + "schwarz": "black", + "черный": "black", + "weiß": "white", + "weiss": "white", + "белый": "white", + "серый": "gray", + "grau": "gray", + "silber": "silver", + "сереб": "silver", + "rot": "red", + "красный": "red", + "blau": "blue", + "синий": "blue", + "grün": "green", + "gruen": "green", + "зеленый": "green", +} + + +class MobileDeMapper: + """Map mobile.de search/detail payloads into the existing CarRecord schema.""" + + def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord: + raw = listing.raw or {} + attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {} + make = raw.get("make") if isinstance(raw.get("make"), dict) else {} + model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {} + + brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN") + model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN") + origin_id = self.origin_id(str(listing.id)) + title = " ".join(part for part in [listing.title, listing.subtitle] if part) + + return CarRecord( + parser_id=self._parser_id(origin_id), + brand=brand[:50] or "UNKNOWN", + model=model[:50] or "UNKNOWN", + year=self._year_from_first_registration(listing.first_registration or attr.get("fr")), + price=self._money_to_int(listing.price or raw.get("p")), + currency="EUR", + mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0, + country="NA", + is_sold=False, + color=self._normalize_color(attr.get("ecol")), + drive=None, + gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")), + steering_wheel="LEFT", + body_type=self._normalize_body(attr.get("c")), + engine_volume=self._int_from_text(attr.get("cc")), + selling_type="CLASSIFIED", + one_owner=(str(attr.get("pvo") or "").strip() == "1"), + new_car=False, + is_hidden=False, + origin="MOBILE_DE", + origin_url=listing.url, + origin_id=origin_id, + is_damaged=bool(raw.get("hasDamage")), + evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None, + non_smoking=True, + rental=False, + repair_history=bool(raw.get("hasDamage")), + slug=self._slugify(title or f"{brand} {model}"), + last_seen_at=datetime.now(timezone.utc), + images=self._images_from_listing(raw), + ) + + def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord: + title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN") + subtitle = self._text(detail.get("subTitle")) + fake_listing = MobileDeListing( + id=str(listing_id), + url=MobileDeClient.build_detail_url(listing_id), + title=title, + subtitle=subtitle, + price=self._text(detail.get("price") or detail.get("p")), + raw=detail, + ) + return self.listing_to_car_record(fake_listing) + + @staticmethod + def origin_id(listing_id: str) -> str: + return f"mobile.de:{listing_id}" + + @staticmethod + def _parser_id(origin_id: str) -> str: + digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16] + return f"mobilede-{digest}" + + @staticmethod + def _text(value: Any) -> str: + return "" if value is None else str(value).strip() + + @classmethod + def _money_to_int(cls, value: Any) -> int | None: + return cls._int_from_text(value) + + @staticmethod + def _int_from_text(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(value) + digits = re.sub(r"[^0-9]", "", str(value)) + return int(digits) if digits else None + + @staticmethod + def _year_from_first_registration(value: Any) -> int | None: + text = "" if value is None else str(value) + match = re.search(r"(19|20)\d{2}", text) + return int(match.group(0)) if match else None + + @staticmethod + def _brand_from_title(title: str | None) -> str | None: + if not title: + return None + return title.split()[0] + + @staticmethod + def _model_from_title(title: str | None, brand: str) -> str | None: + if not title: + return None + rest = title.replace(brand, "", 1).strip() + return rest or None + + @staticmethod + def _normalize_gearbox(value: Any) -> str | None: + text = "" if value is None else str(value).lower() + for marker, mapped in _GEARBOX_MAP.items(): + if marker in text: + return mapped + return None + + @staticmethod + def _normalize_body(value: Any) -> str: + text = "" if value is None else str(value).lower() + for marker, mapped in _BODY_MAP.items(): + if marker in text: + return mapped + return "OTHER" + + @staticmethod + def _normalize_color(value: Any) -> str: + text = "" if value is None else str(value).lower().strip() + for marker, mapped in _COLOR_MAP.items(): + if marker in text: + return mapped + return text[:50] if text else "other" + + @staticmethod + def _slugify(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-") + return slug[:180] or "mobilede-car" + + @staticmethod + def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]: + urls: list[str] = [] + image = raw.get("image") + if isinstance(image, str): + urls.append(MobileDeMapper._normalize_image_url(image)) + images = raw.get("images") + if isinstance(images, list): + for item in images: + if isinstance(item, str): + urls.append(MobileDeMapper._normalize_image_url(item)) + elif isinstance(item, dict): + src = item.get("src") or item.get("url") or item.get("uri") + if src: + urls.append(MobileDeMapper._normalize_image_url(str(src))) + return [ + ImageRecord(fullres_image=url, preview_image=url, order_index=index) + for index, url in enumerate(dict.fromkeys(url for url in urls if url)) + ] + + @staticmethod + def _normalize_image_url(value: str) -> str: + url = str(value).strip() + if not url: + return "" + if url.startswith("//"): + return f"https:{url}" + if url.startswith("http://") or url.startswith("https://"): + return url + return f"https://{url.lstrip('/')}" diff --git a/iaai_scraper/mobile_de/models.py b/iaai_scraper/mobile_de/models.py new file mode 100644 index 0000000..9f5b573 --- /dev/null +++ b/iaai_scraper/mobile_de/models.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class MobileDeListing: + """One listing extracted from mobile.de search results.""" + + id: str + url: str + title: str | None = None + subtitle: str | None = None + price: str | None = None + seller_name: str | None = None + seller_type: str | None = None + location: str | None = None + first_registration: str | None = None + mileage: str | None = None + power: str | None = None + fuel: str | None = None + transmission: str | None = None + raw: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class MobileDeSearchPage: + """Parsed mobile.de search page.""" + + url: str + page_number: int + total_results: int | None + listings: list[MobileDeListing] + raw_search_results: dict[str, Any] = field(default_factory=dict) diff --git a/iaai_scraper/mobile_de/scraper.py b/iaai_scraper/mobile_de/scraper.py new file mode 100644 index 0000000..3135a36 --- /dev/null +++ b/iaai_scraper/mobile_de/scraper.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import logging +import os +from collections.abc import Callable +from dataclasses import asdict +from typing import Any + +from ..core.config import Settings, settings +from ..storage.db import PersistenceService +from .client import MobileDeClient +from .mapper import MobileDeMapper +from .models import MobileDeListing + +logger = logging.getLogger("mobile_de.scraper") + +MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2"))) +MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1"))) + + +class MobileDeScraper: + """High-level mobile.de scraper facade.""" + + def __init__( + self, + client: MobileDeClient | None = None, + persistence: PersistenceService | None = None, + mapper: MobileDeMapper | None = None, + runtime_settings: Settings | None = None, + ) -> None: + self.settings = runtime_settings or settings + self.client = client or MobileDeClient() + self.persistence = persistence or PersistenceService(self.settings) + self.mapper = mapper or MobileDeMapper() + + def collect_search( + self, + *, + start_page: int = 1, + max_pages: int = 1, + search_url: str | None = None, + make_id: str | None = None, + model_id: str | None = None, + price_min: str | None = None, + price_max: str | None = None, + year_min: str | None = None, + year_max: str | None = None, + mileage_min: str | None = None, + mileage_max: str | None = None, + sort_by: str | None = None, + sort_order: str | None = None, + progress_callback: Callable[[str, dict[str, Any]], None] | None = None, + ) -> dict[str, Any]: + params: dict[str, str | None] = {} + if not search_url: + params = { + "p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None, + "fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None, + "ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None, + } + if make_id: + params["ms"] = self.client.build_make_model_param(make_id, model_id) + if sort_by: + params["sb"] = sort_by + if sort_order: + params["od"] = sort_order + + logger.debug( + "mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s", + start_page, + max_pages, + bool(search_url), + make_id, + model_id, + year_min, + year_max, + price_min, + price_max, + mileage_min, + mileage_max, + ) + pages = [] + + def _on_page(page, meta: dict[str, int | None]) -> None: + payload = { + **meta, + "page_url": page.url, + "unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id}) + + len({listing.id for listing in page.listings if listing.id}), + } + if progress_callback is not None: + progress_callback("page_collected", payload) + + concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1"))) + if concurrent_pages > 1 and max_pages and max_pages > 1: + pages.extend(self.client.fetch_search_pages_concurrent( + start_page=start_page, + max_pages=max_pages, + workers=concurrent_pages, + search_url=search_url, + progress_callback=_on_page, + **params, + )) + else: + for page in self.client.iter_search_pages( + start_page=start_page, + max_pages=max_pages, + search_url=search_url, + progress_callback=_on_page, + **params, + ): + pages.append(page) + + unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id}) + result = { + "source": "mobile.de", + "strategy_note": ( + "mobile.de UI shows 50 pages, but pageNumber works beyond 50; " + "for full coverage split by make/model/year/price and deduplicate by id." + ), + "search_url": search_url, + "pages": [asdict(page) for page in pages], + "listing_count": sum(len(page.listings) for page in pages), + "unique_listing_count": len(unique_ids), + "unique_listing_ids": unique_ids, + } + logger.debug( + "mobile.de collect_search finished: pages=%s listings=%s unique=%s", + len(pages), + result["listing_count"], + result["unique_listing_count"], + ) + if progress_callback is not None: + progress_callback( + "search_collection_done", + { + "pages_collected": len(pages), + "listing_count": result["listing_count"], + "unique_listing_count": result["unique_listing_count"], + }, + ) + return result + + def collect_detail(self, listing_id: str) -> dict[str, Any]: + return { + "source": "mobile.de", + "listing_id": str(listing_id), + "url": self.client.build_detail_url(listing_id), + "listing": self.client.fetch_detail(listing_id), + } + + def init_db(self) -> dict[str, Any]: + self.persistence.create_tables() + return {"status": "ok", "source": "mobile.de"} + + def sync_search( + self, + *, + start_page: int = 1, + max_pages: int = 1, + lane: str = "mobile_de_cars", + only_new: bool | None = None, + search_url: str | None = None, + make_id: str | None = None, + model_id: str | None = None, + price_min: str | None = None, + price_max: str | None = None, + year_min: str | None = None, + year_max: str | None = None, + mileage_min: str | None = None, + mileage_max: str | None = None, + sort_by: str | None = None, + sort_order: str | None = None, + progress_callback: Callable[[str, dict[str, Any]], None] | None = None, + ) -> dict[str, Any]: + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane) + failed = 0 + logger.debug( + "mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s", + run_id, + lane, + start_page, + max_pages, + ) + try: + data = self.collect_search( + start_page=start_page, + max_pages=max_pages, + search_url=search_url, + make_id=make_id, + model_id=model_id, + price_min=price_min, + price_max=price_max, + year_min=year_min, + year_max=year_max, + mileage_min=mileage_min, + mileage_max=mileage_max, + sort_by=sort_by, + sort_order=sort_order, + progress_callback=progress_callback, + ) + # Map serialized listings back to records. + records = [] + for page in data["pages"]: + for item in page["listings"]: + records.append(self.mapper.listing_to_car_record(MobileDeListing(**item))) + + skipped_existing = 0 + if only_new and records: + existing_origin_ids = self.persistence.get_existing_origin_ids( + [record.origin_id for record in records if record.origin_id] + ) + before_filter_count = len(records) + + # For newest-first, cut tail after existing streak. + should_cut_tail = ( + str(sort_by or "").lower() == "doc" + and str(sort_order or "").lower() == "down" + and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0 + ) + if should_cut_tail: + filtered_records = [] + existing_streak = 0 + considered = 0 + for record in records: + considered += 1 + is_existing = record.origin_id in existing_origin_ids + if is_existing: + skipped_existing += 1 + existing_streak += 1 + if ( + existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK + and len(filtered_records) >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS + ): + break + continue + existing_streak = 0 + filtered_records.append(record) + records = filtered_records + data["listing_count"] = considered + data["unique_listing_count"] = min(int(data.get("unique_listing_count", considered)), considered) + logger.info( + "mobile.de only_new head-cut applied: considered=%s kept=%s skipped_existing=%s streak=%s", + considered, + len(records), + skipped_existing, + MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK, + ) + else: + records = [record for record in records if record.origin_id not in existing_origin_ids] + skipped_existing = before_filter_count - len(records) + logger.info( + "mobile.de only_new filter applied: kept=%s skipped_existing=%s", + len(records), + skipped_existing, + ) + + if progress_callback is not None: + progress_callback( + "records_mapped", + { + "record_count": len(records), + "skipped_existing": skipped_existing, + "only_new": bool(only_new), + "run_id": run_id, + }, + ) + + upsert = self.persistence.upsert_cars_batch(records) if records else { + "inserted": 0, + "updated": 0, + "images_upserted": 0, + } + logger.debug( + "mobile.de sync_search upsert finished: run_id=%s inserted=%s updated=%s images=%s", + run_id, + upsert.get("inserted", 0), + upsert.get("updated", 0), + upsert.get("images_upserted", 0), + ) + if progress_callback is not None: + progress_callback( + "db_upsert_done", + { + "run_id": run_id, + "inserted": int(upsert.get("inserted", 0)), + "updated": int(upsert.get("updated", 0)), + "images_upserted": int(upsert.get("images_upserted", 0)), + }, + ) + self.persistence.finish_sync_run( + run_id, + status="success", + ids_fetched=len(records), + cars_upserted=int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0)), + cars_failed=failed, + images_upserted=int(upsert.get("images_upserted", 0)), + ) + logger.debug( + "mobile.de sync_search completed: run_id=%s listings=%s unique=%s", + run_id, + data.get("listing_count", 0), + data.get("unique_listing_count", 0), + ) + return {"run_id": run_id, "source": "mobile.de", "upsert": upsert, "skipped_existing": skipped_existing, **data} + except Exception as exc: + self.persistence.finish_sync_run( + run_id, + status="failed", + ids_fetched=0, + cars_upserted=0, + cars_failed=failed, + images_upserted=0, + error_summary=str(exc), + ) + logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True) + raise + + def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]: + self.persistence.create_tables() + detail = self.client.fetch_detail(listing_id) + record = self.mapper.detail_to_car_record(str(listing_id), detail) + result = self.persistence.upsert_car(record) + return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result} diff --git a/iaai_scraper/parsing/__init__.py b/iaai_scraper/parsing/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/parsing/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/parsing/fast_mapper.py b/iaai_scraper/parsing/fast_mapper.py new file mode 100644 index 0000000..80d8843 --- /dev/null +++ b/iaai_scraper/parsing/fast_mapper.py @@ -0,0 +1,368 @@ +from __future__ import annotations + +import re +from datetime import datetime, timezone +from typing import Any +from urllib.parse import urlparse + +from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text +from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES +from ..storage.schemas import CarRecord, ImageRecord + +ORIGIN_PREFIX = "iaai:" +ORIGIN_URL_BASE = "https://www.iaai.com/VehicleDetail" +DAMAGE_NEUTRAL_VALUES = { + "NORMAL WEAR & TEAR", + "NORMAL WEAR", + "NORMALWEAR&TEAR", + "NONE", + "NO DAMAGE", + "NO VISIBLE DAMAGE", + "MINOR DENT/SCRATCHES", +} +INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"} + + +class FastCarMapper: + """Maps IAAI ProductDetailsVM payloads directly to project CarRecord.""" + + def map_payload_to_record( + self, + *, + detail_payload: dict[str, Any], + vehicle_url: str | None = None, + listing_vehicle: FastListingVehicle | None = None, + ) -> CarRecord: + inventory_view = detail_payload.get("inventoryView") + if not isinstance(inventory_view, dict): + raise RuntimeError("detail payload missing inventoryView") + attributes = inventory_view.get("attributes") + if not isinstance(attributes, dict): + raise RuntimeError("detail payload missing inventoryView.attributes") + + inventory_id = parse_text(attributes.get("Id")) + if not inventory_id and listing_vehicle is not None: + inventory_id = listing_vehicle.inventory_id + if not inventory_id and vehicle_url: + inventory_id = self._inventory_id_from_url(vehicle_url) + if not inventory_id: + raise RuntimeError("missing inventory id") + + brand = self._limit_text(parse_text(attributes.get("Make")) or "UNKNOWN", 50) + model = self._build_model_name(parse_text(attributes.get("Model")), parse_text(attributes.get("Series"))) or "UNKNOWN" + model = self._limit_text(model, 50) + year = self._parse_year(attributes.get("Year")) + + auction_info = detail_payload.get("auctionInformation") + auction_info = auction_info if isinstance(auction_info, dict) else {} + bidding_info = auction_info.get("biddingInformation") + bidding_info = bidding_info if isinstance(bidding_info, dict) else {} + prebid_info = auction_info.get("prebidInformation") + prebid_info = prebid_info if isinstance(prebid_info, dict) else {} + + high_bid = self._first_positive_int( + prebid_info.get("decimalHighBidAmount"), + prebid_info.get("highBidAmount"), + bidding_info.get("highBidAmount"), + ) + buy_now = self._first_positive_int( + bidding_info.get("buyNowAmount"), + prebid_info.get("buyNowPrice"), + bidding_info.get("buyNowPrice"), + ) + price = high_bid if high_bid is not None else buy_now + + image_dimensions = inventory_view.get("imageDimensions") + image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {} + keys_container = image_dimensions.get("keys") + keys_container = keys_container if isinstance(keys_container, dict) else {} + image_keys = keys_container.get("$values") + image_keys = image_keys if isinstance(image_keys, list) else [] + images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)] + + primary_damage = parse_text(attributes.get("PrimaryDamageDesc")) + secondary_damage = parse_text(attributes.get("SecondaryDamageDesc")) + origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}" + normalized_origin_id = self._normalize_origin_inventory_id(inventory_id) + origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}" + + return CarRecord( + parser_id=self._generate_parser_id(origin_id), + brand=brand, + model=model, + year=year, + price=price, + currency=self._map_currency(parse_text(attributes.get("Currency")) or (listing_vehicle.currency if listing_vehicle else None)), + mileage=self._parse_non_negative_int(attributes.get("ODOValue")) or 0, + country=self._map_country(inventory_id), + is_sold=self._is_sold(listing_vehicle), + color=self._normalize_color(parse_text(attributes.get("ExteriorColor")) or parse_text(attributes.get("ColorDesc"))), + drive=self._map_drive(parse_text(attributes.get("DriveLineTypeDesc"))), + gearbox=self._map_gearbox(parse_text(attributes.get("Transmission"))), + steering_wheel="LEFT", + body_type=self._map_body_type(parse_text(attributes.get("BodyStyleName")) or parse_text(attributes.get("VehicleClass"))), + engine_volume=self._parse_engine_volume(parse_text(attributes.get("EngineSize")) or parse_text(attributes.get("EngineInformation"))), + selling_type="AUCTION", + one_owner=False, + new_car=False, + is_hidden=not bool(images), + origin="IAAI", + origin_url=origin_url, + origin_id=origin_id, + is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage), + evaluation=parse_text(attributes.get("VehicleGrade")), + non_smoking=True, + rental=False, + repair_history=False, + slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))), + last_seen_at=datetime.now(timezone.utc), + images=images, + ) + + def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]: + inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {} + inventory_view = inventory_view if isinstance(inventory_view, dict) else {} + attr = inventory_view.get("attributes") + attr = attr if isinstance(attr, dict) else {} + auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {} + auction_info = auction_info if isinstance(auction_info, dict) else {} + bidding_info = auction_info.get("biddingInformation") + bidding_info = bidding_info if isinstance(bidding_info, dict) else {} + prebid_info = auction_info.get("prebidInformation") + prebid_info = prebid_info if isinstance(prebid_info, dict) else {} + image_dimensions = inventory_view.get("imageDimensions") + image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {} + keys_container = image_dimensions.get("keys") + keys_container = keys_container if isinstance(keys_container, dict) else {} + image_keys = keys_container.get("$values") + image_keys = image_keys if isinstance(image_keys, list) else [] + image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)] + return { + "source_url": vehicle_url, + "lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"), + "year": attr.get("Year"), + "make": attr.get("Make"), + "model": attr.get("Model"), + "trim": attr.get("Series"), + "body_type": attr.get("BodyStyleName"), + "drive": attr.get("DriveLineTypeDesc"), + "engine": attr.get("EngineInformation") or attr.get("EngineSize"), + "fuel_type": attr.get("FuelTypeCode"), + "gearbox": attr.get("Transmission"), + "color": attr.get("ExteriorColor"), + "primary_damage": attr.get("PrimaryDamageDesc"), + "secondary_damage": attr.get("SecondaryDamageDesc"), + "odometer": attr.get("ODOValue"), + "location": attr.get("BranchName"), + "auction_date": attr.get("AuctionDateTime"), + "title": attr.get("Title"), + "current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"), + "buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"), + "actual_cash_value": attr.get("ProviderACV"), + "estimated_repair_cost": attr.get("EstRepairCost"), + "image_urls": image_urls, + } + + @staticmethod + def _inventory_id_from_url(vehicle_url: str) -> str | None: + tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] + return tail or None + + @staticmethod + def _normalize_origin_inventory_id(inventory_id: str) -> str: + return inventory_id.strip().split("~", 1)[0] + + @staticmethod + def _build_model_name(model: str | None, series: str | None) -> str: + unique_parts: list[str] = [] + seen: set[str] = set() + for value in (model, series): + if not value: + continue + normalized = " ".join(value.split()) + key = normalized.casefold() + if key in seen: + continue + seen.add(key) + unique_parts.append(normalized) + return " ".join(unique_parts).strip() + + @staticmethod + def _parse_year(value: Any) -> int | None: + parsed = parse_int(value) + if parsed is None or parsed < 1900 or parsed > 2100: + return None + return parsed + + @staticmethod + def _parse_non_negative_int(value: Any) -> int | None: + parsed = parse_int(value) + if parsed is None or parsed < 0: + return None + return parsed + + @classmethod + def _first_positive_int(cls, *values: Any) -> int | None: + for value in values: + parsed = cls._parse_non_negative_int(value) + if parsed is not None and parsed > 0: + return parsed + return None + + @staticmethod + def _normalize_color(value: str | None) -> str: + if not value: + return "other" + normalized = value.strip().lower() + if "/" in normalized: + normalized = normalized.split("/", 1)[0].strip() + return normalized or "other" + + @staticmethod + def _map_currency(value: str | None) -> str: + normalized = (value or "USD").strip().upper() + return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD" + + @staticmethod + def _map_country(inventory_id: str) -> str: + upper_id = inventory_id.strip().upper() + if upper_id.endswith("~CA"): + return "CA" + if upper_id.endswith("~US"): + return "US" + return "NA" + + @staticmethod + def _map_drive(value: str | None) -> str | None: + if not value: + return None + normalized = value.strip().lower() + candidates: tuple[str, ...] | None = None + if "front" in normalized or normalized == "fwd": + candidates = ("FWD",) + elif "all wheel" in normalized or "4x4" in normalized or normalized == "awd" or "four wheel" in normalized: + candidates = ("4WD", "FOUR_WD") + elif "rear" in normalized or normalized == "rwd": + candidates = ("RWD",) + elif "2wd" in normalized or "two wheel" in normalized: + candidates = ("2WD", "TWO_WD") + elif "unknown" in normalized or normalized in {"na", "n/a"}: + candidates = ("NA",) + return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None + + @staticmethod + def _map_gearbox(value: str | None) -> str | None: + if not value: + return None + normalized = value.strip().lower() + candidates: tuple[str, ...] | None = None + if "cvt" in normalized: + candidates = ("CVT",) + elif "manual" in normalized or normalized == "mt": + candidates = ("MT",) + elif "electric" in normalized or normalized == "ev": + candidates = ("EV",) + elif "auto" in normalized or normalized == "at": + candidates = ("AT",) + elif "unknown" in normalized or normalized in {"na", "n/a"}: + candidates = ("NA",) + return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None + + @staticmethod + def _map_body_type(value: str | None) -> str: + if not value: + return "OTHER" + normalized = value.strip().lower() + candidates: tuple[str, ...] | None = None + if "sedan" in normalized: + candidates = ("SEDAN",) + elif "sport utility" in normalized or normalized == "suv" or "crossover" in normalized: + candidates = ("SUV",) + elif "hatch" in normalized: + candidates = ("HATCHBACK",) + elif "wagon" in normalized: + candidates = ("STATION_WAGON", "Station Wagon") + elif "coupe" in normalized: + candidates = ("COUPE",) + elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized): + candidates = ("PICKUP", "Pickup") + elif "convertible" in normalized or "roadster" in normalized or "cabrio" in normalized: + candidates = ("OPEN", "Open") + elif "van" in normalized: + candidates = ("MINIVAN",) + elif "truck" in normalized or "chassis" in normalized: + candidates = ("TRUCK", "Truck") + elif "rv" in normalized or "motorized" in normalized: + candidates = ("RV",) + elif normalized in {"other", "unknown"}: + candidates = ("OTHER", "Other") + return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER" + + @staticmethod + def _parse_engine_volume(value: str | None) -> int | None: + if not value: + return None + match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", value) + if not match: + return None + try: + liters = float(match.group(1)) + except ValueError: + return None + cc = int(round(liters * 1000)) + if cc <= 0 or cc > 10000: + return None + return cc + + @staticmethod + def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool: + neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES} + for value in (primary_damage, secondary_damage): + if not value: + continue + normalized = value.replace(" ", "").strip().upper() + if normalized and normalized not in neutral: + return True + return False + + @staticmethod + def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool: + if listing_vehicle is None: + return False + if listing_vehicle.timed_auction_closed: + return True + status = (listing_vehicle.inventory_status or "").strip().upper() + return status in INACTIVE_STATUS_VALUES + + @staticmethod + def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None: + if not candidates: + return fallback if fallback in allowed else None + allowed_set = set(allowed) + for candidate in candidates: + if candidate in allowed_set: + return candidate + return fallback if fallback in allowed_set else None + + @staticmethod + def _limit_text(value: str, max_length: int) -> str: + return value if len(value) <= max_length else value[:max_length].rstrip() + + @staticmethod + def _slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" + + @staticmethod + def _generate_parser_id(origin_id: str) -> str: + import hashlib + from string import ascii_letters, digits + + digest = hashlib.sha256(origin_id.encode()).digest() + alphabet = ascii_letters + digits + base = len(alphabet) + num = int.from_bytes(digest[:17], "big") + chars: list[str] = [] + for _ in range(22): + num, idx = divmod(num, base) + chars.append(alphabet[idx]) + return "car-" + "".join(chars) diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py new file mode 100644 index 0000000..b31309b --- /dev/null +++ b/iaai_scraper/parsing/mapper.py @@ -0,0 +1,405 @@ +import hashlib +import re +from datetime import datetime, timezone +from string import ascii_letters, digits +from typing import Any +from urllib.parse import urlparse + +from ..core.utils import first_non_empty +from ..storage.enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) +from ..storage.schemas import CarRecord, ImageRecord + + +class CarMapper: + # Маппер IAAI в CarRecord. + + BODY_MAP = { + "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", + "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", + "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", + "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", + "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", + "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", + } + DRIVE_MAP = { + "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", + "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", + "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", + } + GEARBOX_MAP = { + "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", + "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", + "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", + } + STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} + COUNTRY_MAP = { + "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", + "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", + } + NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} + + def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: + # Собираем DB-модель. + vehicle_summary = vehicle_summary or {} + payload_insights = payload_insights or {} + core = payload_insights.get("vehicle_core", {}) + pricing = payload_insights.get("pricing", {}) + damage = payload_insights.get("damage", {}) + auction = payload_insights.get("auction", {}) + images = payload_insights.get("images", {}) + + origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) + parser_id = self._generate_parser_id(origin_id) + brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" + model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" + year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")])) + price = self._first_parsed_int( + [ + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + pricing.get("actual_cash_value"), + ], + self._to_money_int, + ) + mileage = self._parse_odometer( + first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")]) + ) + color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) + drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) + # Пробуем взять привод из двигателя. + if not drive or drive == "NA": + engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")])) + if engine_text: + inferred_drive = self._normalize_drive(engine_text) + if inferred_drive and inferred_drive != "NA": + drive = inferred_drive + gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) + steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" + body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) + engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")])) + title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) + seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) + location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""])) + country = self._normalize_country(first_non_empty([core.get("country"), location, "US"])) + is_damaged = self._bool_damage(damage, vehicle_summary) + is_sold = self._bool_sold(auction) + one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) + new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) + rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) + repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) + non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) + evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None + currency = self._normalize_currency( + first_non_empty( + [ + pricing.get("currency"), + vehicle_summary.get("currency"), + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + "USD", + ] + ) + ) + slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))) + images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) + origin = "IAAI" + + return CarRecord( + parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, + mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, + steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, + selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car, + is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, + evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, + slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records, + ) + + @staticmethod + def _as_str(value: Any) -> str: + return "" if value is None else str(value).strip() + + @staticmethod + def _to_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + digits = re.sub(r"[^\d]", "", str(value)) + return int(digits) if digits else None + + @classmethod + def _to_money_int(cls, value: Any) -> int | None: + # Нормализация цены. + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return int(value) + + text = str(value).strip() + if not text: + return None + + lowered = text.lower() + if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]): + return None + + numbers = re.findall(r"\d[\d\s.,]*", text) + if not numbers: + return None + + best: int | None = None + for number in numbers: + clean = number.replace(" ", "") + if "," in clean and "." in clean: + # Поддержка двух форматов. + if clean.rfind(",") > clean.rfind("."): + clean = clean.replace(".", "").replace(",", ".") + else: + clean = clean.replace(",", "") + elif "," in clean: + parts = clean.split(",") + # Десятичный или тысячный разделитель. + if len(parts[-1]) in {1, 2} and len(parts) == 2: + clean = clean.replace(",", ".") + else: + clean = clean.replace(",", "") + elif "." in clean: + parts = clean.split(".") + if not (len(parts[-1]) in {1, 2} and len(parts) == 2): + clean = clean.replace(".", "") + + try: + parsed = int(float(clean)) + except ValueError: + continue + + if parsed > 0 and (best is None or parsed > best): + best = parsed + + return best + + @staticmethod + def _first_parsed_int(values: list[Any], parser) -> int | None: + for value in values: + parsed = parser(value) + if parsed is not None: + return parsed + return None + + @staticmethod + def _to_year(value: Any) -> int | None: + parsed = CarMapper._to_int(value) + if parsed is None: + return None + if 1900 <= parsed <= 2100: + return parsed + return None + + @staticmethod + def _parse_odometer(value: Any) -> int: + # Разбор пробега. + if value is None: + return 0 + text = str(value).strip() + if not text: + return 0 + lowered = text.lower() + if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]): + return 0 + # Ищем число. + numbers = re.findall(r"[\d,]+", text) + for num_str in numbers: + clean = num_str.replace(",", "") + if clean.isdigit() and int(clean) > 0: + return int(clean) + return 0 + + def _to_engine_cc(self, value: Any) -> int | None: + # Поддержка литров и cc. + text = str(value).lower().strip() if value is not None else "" + if not text: + return None + if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): + return int(float(liters_match.group(1)) * 1000) + if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text): + return int(cubic_match.group(1)) + return int(text) if re.match(r"^\d+$", text) else None + + def _normalize_currency(self, value: Any) -> str: + text = self._as_str(value) + upper = text.upper() or "USD" + if any(token in text for token in ["€", "EUR"]): + return "EUR" + if any(token in text for token in ["¥", "JPY"]): + return "JPY" + if any(token in text for token in ["₩", "KRW"]): + return "KRW" + if any(token in text for token in ["£", "GBP"]): + return "GBP" + if any(token in text for token in ["₽", "RUB"]): + return "RUB" + if any(token in text for token in ["AED", "د.إ"]): + return "AED" + if any(token in text for token in ["CA$", "CAD"]): + return "CAD" + + text = upper + if text in CURRENCY_ENUM_VALUES: + return text + return "USD" if "$" in str(value) else "USD" + + def _normalize_drive(self, value: Any) -> str | None: + return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_gearbox(self, value: Any) -> str | None: + return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_steering(self, value: Any) -> str | None: + return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) + + def _normalize_body_type(self, value: Any) -> str: + return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" + + def _normalize_country(self, value: Any) -> str: + fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA" + return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback + + def _normalize_selling_type(self, value: Any) -> str: + text = self._as_str(value) or "AUCTION" + return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" + + def _map_value( + self, + value: Any, + mapping: dict[str, str], + allowed_values: tuple[str, ...], + *, + empty_default: str | None, + fallback: str | None, + ) -> str | None: + # Общая нормализация enum. + text = self._as_str(value).lower() + if not text: + return empty_default + if (mapped := mapping.get(text)) and mapped in allowed_values: + return mapped + for marker, mapped in mapping.items(): + if marker in text and mapped in allowed_values: + return mapped + return fallback + + @staticmethod + def _normalize_color(value: Any) -> str: + text = str(value).strip() if value is not None else "other" + if not text: + return "other" + if "/" in text: + text = text.split("/")[0].strip() + return text.lower() or "other" + + @staticmethod + def _boolish(value: Any) -> bool: + return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} + + def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: + for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: + text = self._as_str(value).lower() + if text and text not in self.NO_DAMAGE_MARKERS: + return True + return False + + def _bool_sold(self, auction: dict[str, Any]) -> bool: + return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) + + def _build_images(self, urls: list[Any]) -> list[ImageRecord]: + # Для imageKeys берём самый большой размер. + best_by_key: dict[str, str] = {} + key_order: list[str] = [] + non_keyed: list[str] = [] + for item in urls: + if not isinstance(item, str): + continue + url = item.strip() + if not url: + continue + if m := re.search(r'imageKeys=([^&]+)', url): + img_key = m.group(1) + w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 + existing = best_by_key.get(img_key) + if existing is None: + best_by_key[img_key] = url + key_order.append(img_key) + else: + existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 + if w > existing_w: + best_by_key[img_key] = url + elif url not in non_keyed: + non_keyed.append(url) + deduped = [best_by_key[k] for k in key_order] + non_keyed + return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] + + @staticmethod + def _make_fullres_url(url: str) -> str: + if "vis.iaai.com" in url: + return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) + return url + + @staticmethod + def _make_preview_url(url: str) -> str: + if "vis.iaai.com" in url: + preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) + if preview != url: + return preview + return url + + # Формирование parser_id. + + _PARSER_ID_ALPHABET = ascii_letters + digits + + @classmethod + def _generate_parser_id(cls, origin_id: str) -> str: + # Стабильный parser_id. + digest = hashlib.sha256(origin_id.encode()).digest() + alphabet = cls._PARSER_ID_ALPHABET + base = len(alphabet) + num = int.from_bytes(digest[:17], "big") # Хватает на 22 символа. + chars: list[str] = [] + for _ in range(22): + num, idx = divmod(num, base) + chars.append(alphabet[idx]) + return "car-" + "".join(chars) + + def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: + # Формат: iaai:{lot_number}. + for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]: + text = self._as_str(value) + if text: + return f"iaai:{text}" + tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] + if "~" in tail: + tail = tail.split("~")[0] + raw = tail or self._slugify(vehicle_url) + return f"iaai:{raw}" + + @staticmethod + def _slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" + + diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py new file mode 100644 index 0000000..4d3fee8 --- /dev/null +++ b/iaai_scraper/parsing/parser.py @@ -0,0 +1,408 @@ +import html as html_module +import json +import logging +import re +from typing import Any + +from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty + +logger = logging.getLogger("iaai_scraper.parsers") + + +class VehicleParser: + # Парсер страницы авто. + + # Регулярки парсинга и защиты. + _BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE) + _CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]) + _ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"]) + _CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE) + _ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE) + + SUMMARY_KEY_MAP = { + "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, + "year": {"year"}, + "make": {"make", "manufacturer", "brand"}, + "model": {"model"}, + "trim": {"trim", "series"}, + "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, + "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, + "secondary_damage": {"secondarydamage"}, + "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, + "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, + "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, + "actual_cash_value": {"actualcashvalue", "acv"}, + "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, + "keys": {"keys", "keystatus"}, + "title": {"titletype", "title", "documenttype"}, + "seller": {"seller", "sellername"}, + "location": {"location", "branchname", "auctionlocation", "branch"}, + "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, + "body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"}, + "drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"}, + "gearbox": {"transmission", "gearbox", "transmissiontype"}, + "engine": {"engine", "enginevolume", "enginetype", "enginedescription"}, + "fuel_type": {"fueltype", "fuel"}, + "cylinders": {"cylinders", "cylindercount"}, + "color": {"color", "primarycolor", "exteriorcolor"}, + } + + DOM_LABEL_MAP: dict[str, str] = { + "stock #": "lot_number", + "stock": "lot_number", + "primary damage": "primary_damage", + "secondary damage": "secondary_damage", + "odometer": "odometer", + "odometer (miles)": "odometer", + "mileage": "odometer", + "body style": "body_type", + "body type": "body_type", + "vehicle type": "body_type", + "engine": "engine", + "engine type": "engine", + "transmission": "gearbox", + "drive line type": "drive", + "driveline type": "drive", + "drive line": "drive", + "driveline": "drive", + "drive type": "drive", + "fuel type": "fuel_type", + "fuel": "fuel_type", + "cylinders": "cylinders", + "exterior/interior": "color", + "exterior color": "color", + "color": "color", + "model": "model", + "series": "trim", + "selling branch": "location", + "vehicle location": "vehicle_location", + "auction date and time": "auction_date", + "sale date": "auction_date", + "lane/run #": "lane", + "actual cash value": "actual_cash_value", + "estimated repair cost": "estimated_repair_cost", + "seller": "seller", + "title/sale doc": "title", + "title/sale doc brand": "title_brand", + "start code": "run_and_drive", + "key": "keys", + "keys": "keys", + "manufactured in": "manufactured_in", + "vehicle class": "vehicle_class", + } + + def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: + result: dict[str, str] = {} + if not dom_text: + return result + lines = [line.strip() for line in dom_text.split("\n") if line.strip()] + known_labels = set(self.DOM_LABEL_MAP.keys()) + skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"} + max_fields = len(set(self.DOM_LABEL_MAP.values())) + + for i, line in enumerate(lines): + # Ранний выход. + if len(result) >= max_fields: + break + + # Метка и значение в одной строке. + colon_pos = line.find(":") + if colon_pos > 0: + label_part = line[:colon_pos].strip().lower() + value_part = line[colon_pos + 1:].strip() + if label_part in known_labels and value_part and value_part.lower() not in skip_values: + field_name = self.DOM_LABEL_MAP[label_part] + if field_name not in result or not result[field_name]: + result[field_name] = value_part + continue + + # Метка и значение в соседних строках. + clean = line.rstrip(":").strip().lower() + clean_alt = clean.rstrip("#").strip() + matched_label = None + if clean in known_labels: + matched_label = clean + elif clean_alt in known_labels: + matched_label = clean_alt + + if matched_label and i + 1 < len(lines): + value = lines[i + 1].strip() + if value.rstrip(":").lower().strip() in known_labels: + continue + if value.lower() in skip_values: + continue + field_name = self.DOM_LABEL_MAP[matched_label] + if field_name not in result or not result[field_name]: + result[field_name] = value + return result + + def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: + result: dict[str, str | None] = {"year": None, "make": None, "model": None} + title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") + if title_match: + result["year"] = title_match.group(1) + result["make"] = title_match.group(2) + result["model"] = title_match.group(3) + return result + dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") + if dom_match: + result["year"] = dom_match.group(1) + result["make"] = dom_match.group(2) + result["model"] = dom_match.group(3).strip() + return result + + def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: + page_html = page_html or "" + dom_text = dom_text or "" + network_dump = network_dump or {} + responses = network_dump.get("json_responses", []) + payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] + dom_kv = self._parse_dom_key_value_pairs(dom_text) + page_title = "" + title_match = re.search(r"]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) + if title_match: + page_title = title_match.group(1).strip() + title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) + + # Один проход по payload. + all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP) + + summary: dict[str, Any] = {"source_url": vehicle_url} + for field, values in all_found.items(): + if field in dom_kv: + values.append(dom_kv[field]) + summary[field] = first_non_empty(values) + + summary["year"] = summary.get("year") or title_parsed.get("year") + summary["make"] = summary.get("make") or title_parsed.get("make") + summary["model"] = summary.get("model") or title_parsed.get("model") + summary["trim"] = summary.get("trim") or dom_kv.get("trim") + summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) + summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) + for dom_field, dom_value in dom_kv.items(): + if dom_field not in summary or not summary[dom_field]: + summary[dom_field] = dom_value + prices = self._extract_prices_from_text(dom_text) + if not summary.get("actual_cash_value") and prices: + summary["actual_cash_value"] = prices[0] + if not summary.get("buy_now"): + buy_now_match = self._BUY_NOW_RE.search(dom_text or "") + if buy_now_match: + summary["buy_now"] = buy_now_match.group(1) + elif prices: + summary["buy_now"] = prices[0] + if not summary.get("current_bid") and len(prices) > 1: + summary["current_bid"] = prices[1] + + embedded = self._extract_embedded_json(page_html) + for item in embedded: + p = item.get("payload") + if isinstance(p, (dict, list)): + payloads.append(p) + # Доп. проход по JSON. + extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP) + for field, vals in extra.items(): + if not summary.get(field): + v = first_non_empty(vals) + if v: + summary[field] = v + + # Передаём готовые image_urls. + image_urls = summary.get("image_urls") or [] + return { + "vehicle_summary": summary, + "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls), + "embedded_json": embedded, + "dom_hints": self._dom_hints(dom_text), + "access_notes": self._build_access_notes(summary, responses), + } + + def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]: + if image_urls is None: + image_urls = self._extract_image_urls(payloads, "", vehicle_url) + return { + "vehicle_core": { + "lot_number": summary.get("lot_number"), "year": summary.get("year"), + "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), + "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), + "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), + "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), + "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), + "color": summary.get("color"), "keys": summary.get("keys"), + }, + "pricing": { + "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), + "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), + "currency": self._guess_currency(summary), + }, + "bids": self._build_bid_insights(summary, payloads), + "damage": { + "primary": summary.get("primary_damage"), + "secondary": summary.get("secondary_damage"), + "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), + }, + "auction": { + "auction_date": summary.get("auction_date"), + "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), + "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), + "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), + "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), + }, + "images": {"count": len(image_urls), "urls": image_urls}, + "source_endpoints": self._build_source_endpoints(responses), + } + + def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: + return { + "amount": summary.get("current_bid"), + "currency": self._guess_currency(summary), + "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), + "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), + } + + def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: + mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} + for item in responses: + url = item.get("url", "") + category = item.get("category", "other") + if category == "vehicle": + mapping["vehicle"].append(url) + lowered = url.lower() + if any(token in lowered for token in ["bid", "offer"]): + mapping["bids"].append(url) + if any(token in lowered for token in ["damage", "report"]): + mapping["damage"].append(url) + if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): + mapping["auction"].append(url) + elif category in mapping: + mapping[category].append(url) + return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} + + def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: + endpoints = [item.get("url", "") for item in responses] + dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None)) + return { + "images_visible": bool(summary.get("image_urls")), + "network_json_count": len(responses), + "possible_captcha": bool(dom_hints.get("has_captcha_text")), + "possible_antibot": bool(dom_hints.get("has_antibot_text")), + "observed_endpoints": endpoints[:20], + } + + @staticmethod + def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: + lowered = {key.lower() for key in keys} + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, lowered)) + return values + + @staticmethod + def _guess_currency(summary: dict[str, Any]) -> str: + for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: + value = str(summary.get(key) or "") + if "$" in value: + return "USD" + if "€" in value: + return "EUR" + if "¥" in value: + return "JPY" + return "USD" + + @staticmethod + def _extract_lot_number(text: str) -> str | None: + match = LOT_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_prices_from_text(text: str) -> list[str]: + return [match.group(1) for match in PRICE_RE.finditer(text or "")] + + @staticmethod + def _extract_embedded_json(html: str) -> list[dict[str, Any]]: + scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) + extracted: list[dict[str, Any]] = [] + for script_text in scripts: + if "{" not in script_text and "[" not in script_text: + continue + # Пропускаем большие блоки. + if len(script_text) > 51_200: + continue + try: + parsed = json.loads(script_text.strip()) + except Exception: + continue + extracted.append({"type": "inline_json", "payload": parsed}) + return extracted + + @staticmethod + def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: + vehicle_key = "" + key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") + if key_match: + vehicle_key = key_match.group(1) + found: list[str] = [] + for payload in payloads: + found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) + flat: list[str] = [] + seen_flat: set[str] = set() + for item in found: + if isinstance(item, str) and item.startswith("http"): + cleaned = html_module.unescape(item) + lowered = cleaned.lower() + if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + elif isinstance(item, list): + for child in item: + if isinstance(child, str) and child.startswith("http"): + cleaned = html_module.unescape(child) + lowered = cleaned.lower() + if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: + for match in re.finditer(pattern, html or "", re.IGNORECASE): + url = html_module.unescape(match.group(1).strip()) + if not url.startswith("http") or url in seen_flat: + continue + lowered = url.lower() + if vehicle_key and vehicle_key in url: + seen_flat.add(url) + flat.append(url) + elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]): + if vehicle_key and vehicle_key not in url: + continue + seen_flat.add(url) + flat.append(url) + if vehicle_key: + for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""): + cleaned = html_module.unescape(url) + if cleaned not in seen_flat and vehicle_key in cleaned: + seen_flat.add(cleaned) + flat.append(cleaned) + filtered: list[str] = [] + for url in flat: + lowered = url.lower() + if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): + continue + if "vis.iaai.com" in lowered and "/resizer" not in lowered: + continue + filtered.append(url) + return filtered + + @staticmethod + def _dom_hints(text: str) -> dict[str, Any]: + lowered = (text or "").lower() + return { + "has_buy_now_text": "buy now" in lowered, + "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, + "has_damage_text": "damage" in lowered, + "has_title_text": "title" in lowered, + "has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS), + "has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS), + } diff --git a/iaai_scraper/proxy_bridge.py b/iaai_scraper/proxy_bridge.py new file mode 100644 index 0000000..251ce58 --- /dev/null +++ b/iaai_scraper/proxy_bridge.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import logging +import os +import select +import socket +import socketserver +import struct +import threading +from urllib.parse import urlsplit + +BUFFER_SIZE = 65536 +CRLF = b"\r\n" +DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1") +DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899")) +SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "") +SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002")) +SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "") +SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "") +RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60")) +MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64")) + +logger = logging.getLogger("proxy_bridge") + + +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + allow_reuse_address = True + daemon_threads = True + + def __init__(self, server_address, request_handler_class): + super().__init__(server_address, request_handler_class) + self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS) + + def process_request_thread(self, request, client_address): + with self._worker_semaphore: + super().process_request_thread(request, client_address) + + +def _recv_exact(sock: socket.socket, size: int) -> bytes: + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise ConnectionError("Unexpected EOF from SOCKS5 server") + data += chunk + return data + + +def _socks5_connect(host: str, port: int) -> socket.socket: + if not SOCKS5_HOST: + raise RuntimeError("SOCKS5_PROXY_HOST is not configured") + + upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30) + upstream.settimeout(30) + + methods = [0x00] + if SOCKS5_USER or SOCKS5_PASS: + methods = [0x02] + upstream.sendall(bytes([0x05, len(methods), *methods])) + version, method = _recv_exact(upstream, 2) + if version != 0x05 or method == 0xFF: + upstream.close() + raise ConnectionError("SOCKS5 authentication negotiation failed") + + if method == 0x02: + username = SOCKS5_USER.encode("utf-8") + password = SOCKS5_PASS.encode("utf-8") + if len(username) > 255 or len(password) > 255: + upstream.close() + raise ValueError("SOCKS5 username/password too long") + upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password) + auth_version, auth_status = _recv_exact(upstream, 2) + if auth_version != 0x01 or auth_status != 0x00: + upstream.close() + raise ConnectionError("SOCKS5 username/password authentication failed") + + try: + socket.inet_aton(host) + addr_type = 0x01 + addr_payload = socket.inet_aton(host) + except OSError: + host_bytes = host.encode("idna") + if len(host_bytes) > 255: + upstream.close() + raise ValueError("Target host is too long for SOCKS5 domain format") + addr_type = 0x03 + addr_payload = bytes([len(host_bytes)]) + host_bytes + + request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port) + upstream.sendall(request) + + response_head = _recv_exact(upstream, 4) + version, reply, _reserved, reply_addr_type = response_head + if version != 0x05 or reply != 0x00: + upstream.close() + raise ConnectionError(f"SOCKS5 connect failed with code {reply}") + + if reply_addr_type == 0x01: + _recv_exact(upstream, 4) + elif reply_addr_type == 0x03: + domain_len = _recv_exact(upstream, 1)[0] + _recv_exact(upstream, domain_len) + elif reply_addr_type == 0x04: + _recv_exact(upstream, 16) + _recv_exact(upstream, 2) + + upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + return upstream + + +def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None: + sockets = [left, right] + left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + try: + while True: + readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS) + if exceptional: + break + if not readable: + logger.debug("Relay idle timeout reached; closing sockets") + return + for current in readable: + other = right if current is left else left + data = current.recv(BUFFER_SIZE) + if not data: + return + other.sendall(data) + finally: + for sock in sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + sock.close() + except OSError: + pass + + +class ProxyHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + try: + request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip() + if not request_line: + return + + method, target, version = request_line.split() + headers = self._read_headers() + logger.info("%s %s", method, target) + + if method.upper() == "CONNECT": + host, port = self._parse_connect_target(target) + logger.info("CONNECT %s:%s", host, port) + upstream = _socks5_connect(host, port) + self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF) + self.wfile.flush() + _relay_bidirectional(self.connection, upstream) + return + + host, port, path = self._parse_forward_target(target, headers) + upstream = _socks5_connect(host, port) + self._send_forward_request(upstream, method, path, version, headers) + body = self._read_request_body(headers) + if body: + upstream.sendall(body) + _relay_bidirectional(self.connection, upstream) + except Exception as exc: + logger.exception("Proxy bridge request failed: %s", exc) + try: + self.wfile.write( + b"HTTP/1.1 502 Bad Gateway" + CRLF + + b"Connection: close" + CRLF + + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF + + b"Bad Gateway" + ) + self.wfile.flush() + except OSError: + pass + + def _read_headers(self) -> list[tuple[str, str]]: + headers: list[tuple[str, str]] = [] + while True: + line = self.rfile.readline(BUFFER_SIZE) + if line in {CRLF, b"\n", b""}: + break + decoded = line.decode("iso-8859-1") + if ":" not in decoded: + continue + name, value = decoded.split(":", 1) + headers.append((name.strip(), value.strip())) + return headers + + @staticmethod + def _parse_connect_target(target: str) -> tuple[str, int]: + if target.startswith("["): + end = target.find("]") + if end == -1 or len(target) <= end + 2 or target[end + 1] != ":": + raise ValueError("Invalid CONNECT target") + host = target[1:end] + port_text = target[end + 2 :] + return host, int(port_text) + + host, port_text = target.rsplit(":", 1) + return host, int(port_text) + + @staticmethod + def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]: + if target.startswith("http://"): + parts = urlsplit(target) + port = parts.port or 80 + path = parts.path or "/" + if parts.query: + path += f"?{parts.query}" + return parts.hostname or "", port, path + + if target.startswith("https://"): + raise ValueError("HTTPS absolute-form request must use CONNECT") + + host_header = next((value for name, value in headers if name.lower() == "host"), "") + if not host_header: + raise ValueError("Missing Host header") + if ":" in host_header: + host, port_text = host_header.rsplit(":", 1) + return host, int(port_text), target + return host_header, 80, target + + def _send_forward_request( + self, + upstream: socket.socket, + method: str, + path: str, + version: str, + headers: list[tuple[str, str]], + ) -> None: + filtered_headers: list[tuple[str, str]] = [] + hop_by_hop = { + "proxy-connection", + "proxy-authorization", + "connection", + "keep-alive", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } + for name, value in headers: + if name.lower() in hop_by_hop: + continue + filtered_headers.append((name, value)) + + request_head = [f"{method} {path} {version}\r\n"] + request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers) + request_head.append("\r\n") + upstream.sendall("".join(request_head).encode("iso-8859-1")) + + def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes: + transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "") + if "chunked" in transfer_encoding.lower(): + return self._read_chunked_request_body() + + content_length = next((value for name, value in headers if name.lower() == "content-length"), None) + if not content_length: + return b"" + return self.rfile.read(int(content_length)) + + def _read_chunked_request_body(self) -> bytes: + chunks: list[bytes] = [] + while True: + size_line = self.rfile.readline(BUFFER_SIZE) + if not size_line: + raise ConnectionError("Unexpected EOF in chunked request") + size_text = size_line.strip().split(b";", 1)[0] + chunk_size = int(size_text, 16) + chunks.append(size_line) + if chunk_size == 0: + while True: + trailer_line = self.rfile.readline(BUFFER_SIZE) + if not trailer_line: + raise ConnectionError("Unexpected EOF in chunked trailers") + chunks.append(trailer_line) + if trailer_line in {CRLF, b"\n"}: + return b"".join(chunks) + + chunk_data = self.rfile.read(chunk_size) + if len(chunk_data) != chunk_size: + raise ConnectionError("Unexpected EOF in chunk body") + chunks.append(chunk_data) + chunk_end = self.rfile.read(2) + if chunk_end != CRLF: + raise ConnectionError("Invalid chunk terminator") + chunks.append(chunk_end) + + +def main() -> None: + if not SOCKS5_HOST: + raise SystemExit("SOCKS5_PROXY_HOST is required") + + logging.basicConfig( + level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(), + format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s", + ) + + with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server: + logger.info( + "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)", + DEFAULT_LISTEN_HOST, + DEFAULT_LISTEN_PORT, + SOCKS5_HOST, + SOCKS5_PORT, + MAX_WORKERS, + ) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py new file mode 100644 index 0000000..cf7a8a4 --- /dev/null +++ b/iaai_scraper/scraper.py @@ -0,0 +1,2660 @@ +import json +import logging +import os +import random +import re +import signal +import time +import uuid +import html as html_module +from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import TimeoutError as FuturesTimeoutError +from datetime import datetime, timezone +from pathlib import Path +from threading import Event, Thread +from typing import Any, Callable +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen + +import urllib3 + +from playwright.sync_api import Error as PlaywrightError +from playwright.sync_api import BrowserContext, Page, sync_playwright +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + +from .browser.fast_client import DETAIL_MARKER, IAAIFastClient, is_challenge_response, parse_product_details_vm +from .browser import BrowserFactory, HumanPacer, NetworkCapture +from .core.config import Settings, settings, parse_listing_segments +from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError +from .core.logs import set_trace_id, setup_logging +from .core.retry import retryable +from .core.runtime_config import RuntimeConfig +from .core.utils import save_to_json +from .fast_sync import FastSyncEngine +from .parsing.fast_mapper import FastCarMapper +from .parsing.mapper import CarMapper +from .parsing.parser import VehicleParser +from .storage.db import PersistenceService +from .browser.listing import ListingCollector, ListingPageResult +from .storage.schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.scraper") +VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) +HTML_TAG_RE = re.compile(r"<[^>]+>") +SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) + +# Таймауты операций страницы. +_PAGE_COLLECT_TIMEOUT_S = 90 +_PAGE_NEXT_TIMEOUT_S = 60 +# Ротация контекста выключена по умолчанию. +_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("IAAI_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) +_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("IAAI_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3) +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120 +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2 + + +class PageOperationTimeoutError(Exception): + """Browser page operation exceeded per-op watchdog timeout.""" + + +class _PageOpWatchdog: + """SIGALRM-based watchdog. + + Работает только в главном потоке процесса (Celery prefork child — это ок). + В других контекстах — no-op. + """ + + def __init__(self, seconds: int, label: str) -> None: + self.seconds = max(1, int(seconds)) + self.label = label + self._old_handler = None + self._active = False + + def _handler(self, signum, frame): # noqa: ARG002 + raise PageOperationTimeoutError( + f"Page operation '{self.label}' exceeded {self.seconds}s watchdog" + ) + + def __enter__(self): + import threading as _threading + if _threading.current_thread() is not _threading.main_thread(): + return self + if not hasattr(signal, "SIGALRM"): + return self + try: + self._old_handler = signal.signal(signal.SIGALRM, self._handler) + signal.alarm(self.seconds) + self._active = True + except (ValueError, OSError): + # signal можно выставлять только из main thread; в остальном пропускаем. + self._active = False + return self + + def __exit__(self, exc_type, exc, tb): + if not self._active: + return False + try: + signal.alarm(0) + if self._old_handler is not None: + signal.signal(signal.SIGALRM, self._old_handler) + except (ValueError, OSError): + pass + return False + + +class _ProgressHeartbeat: + """Периодически пульсует progress во время долгой навигации.""" + + def __init__(self, report_progress: Callable[[str, Any], None], stage: str, interval_s: float = 15.0, **meta: Any) -> None: + self._report_progress = report_progress + self._stage = stage + self._interval_s = max(5.0, float(interval_s)) + self._meta = meta + self._stop = Event() + self._thread: Thread | None = None + + def _run(self) -> None: + while not self._stop.wait(self._interval_s): + self._report_progress(self._stage, **self._meta) + + def __enter__(self): + self._thread = Thread(target=self._run, name=f"progress-heartbeat-{self._stage}", daemon=True) + self._thread.start() + return self + + def __exit__(self, exc_type, exc, tb): + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=1.0) + return False + + +class IAAIScraper: + + @staticmethod + def _is_protection_or_network_error(exc: Exception) -> bool: + message = str(exc).lower() + signals = ( + "captcha", + "antibot", + "blocked", + "challenge", + "ns_error_net_interrupt", + "navigation", + "timeout", + "403", + "429", + ) + return any(signal in message for signal in signals) + + @staticmethod + def _html_to_text(html: str) -> str: + if not html: + return "" + cleaned = SCRIPT_STYLE_RE.sub(" ", html) + text = HTML_TAG_RE.sub(" ", cleaned) + text = html_module.unescape(text) + return re.sub(r"\s+", " ", text).strip() + + @staticmethod + def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None: + dom_hints = parsed.get("dom_hints", {}) or {} + access_notes = parsed.get("access_notes", {}) or {} + summary = parsed.get("vehicle_summary", {}) or {} + + has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model")) + + if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity: + raise AntiBotDetectedError(f"IAAI anti-bot detected for {vehicle_url}") + + if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity: + raise AntiBotDetectedError(f"IAAI blocked or challenged request for {vehicle_url}") + + if not has_identity: + raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") + + @staticmethod + def _extract_origin_id_from_url(vehicle_url: str) -> str | None: + match = VEHICLE_ID_RE.search(vehicle_url) + if not match: + return None + return match.group(1) + + @staticmethod + def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None: + raw_id = IAAIScraper._extract_origin_id_from_url(vehicle_url) + if not raw_id: + return None + return f"iaai:{raw_id}" + + @staticmethod + def _normalize_vehicle_url(vehicle_url: str) -> str: + try: + parts = urlsplit(vehicle_url) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + except Exception: + return vehicle_url + + def __init__(self, runtime_settings: Settings | None = None) -> None: + self.settings = runtime_settings or settings + setup_logging(self.settings.log_level, self.settings.log_file) + self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) + self.trace_id = uuid.uuid4().hex[:12] + set_trace_id(self.trace_id) + self.playwright = None + self.browser = None + self.context: BrowserContext | None = None + self.browser_factory = BrowserFactory(self.settings) + self.pacer = HumanPacer(self.settings) + self.listing_collector = ListingCollector(self.settings, self.pacer) + self.vehicle_parser = VehicleParser() + self.fast_client = IAAIFastClient(self.settings) + self.fast_mapper = FastCarMapper() + self.car_mapper = CarMapper() + self.persistence = PersistenceService(self.settings) + self._shutdown_requested = False + self._progress_callback: Callable[[str, dict], None] | None = None + # HTTP pool: при parallel_tabs=24 и concurrency=4 пик ≈ 96 конкурентных сокетов; + # даём запас до 256, чтобы не упираться в PoolError под всплесками PX/retry. + proxy_url = self.settings.proxy.server + if proxy_url: + _proxy_kwargs: dict = { + "num_pools": 4, + "maxsize": 64, + "block": False, + "retries": False, + "timeout": urllib3.Timeout(connect=5, read=20), + } + if self.settings.proxy.username: + _proxy_kwargs["proxy_headers"] = urllib3.make_headers( + proxy_basic_auth=f"{self.settings.proxy.username}:{self.settings.proxy.password or ''}" + ) + self._http_pool: urllib3.PoolManager = urllib3.ProxyManager(proxy_url, **_proxy_kwargs) + logger.info("HTTP fast-path using proxy: %s", proxy_url) + else: + self._http_pool = urllib3.PoolManager( + num_pools=4, + maxsize=64, + block=False, + retries=False, + timeout=urllib3.Timeout(connect=5, read=20), + ) + + def _new_trace_id(self, prefix: str) -> str: + trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" + self.trace_id = trace_id + set_trace_id(trace_id) + return trace_id + + def _reload_runtime_config(self) -> None: + """Reload runtime config from file to apply include/exclude changes without restart.""" + try: + self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) + except Exception: + logger.warning("Failed to reload runtime config, keeping previous values", exc_info=True) + + def set_progress_callback(self, callback: Callable[[str, dict], None]) -> None: + self._progress_callback = callback + + def _report_progress(self, stage: str, **meta: Any) -> None: + if self._progress_callback is not None: + try: + self._progress_callback(stage, meta) + except Exception: + pass + + def __enter__(self) -> "IAAIScraper": + if self.settings.scraping_profile.http_first and not self.settings.scraping_profile.browser_fallback_enabled: + return self + if self.playwright is None: + self.playwright = sync_playwright().start() + if self.browser is None: + self.browser = self.browser_factory.create_browser(self.playwright) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + def close(self) -> None: + if self.context is not None: + try: + self.context.close() + except PlaywrightError: + pass + finally: + self.context = None + if self.browser is not None: + try: + self.browser.close() + except PlaywrightError: + pass + finally: + self.browser = None + if self.playwright is not None: + try: + self.playwright.stop() + except PlaywrightError: + pass + finally: + self.playwright = None + if getattr(self, "_http_pool", None) is not None: + try: + self._http_pool.clear() + except Exception: + pass + finally: + self._http_pool = None + + def _new_context(self) -> BrowserContext: + if self.browser is None: + self.__enter__() + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = self.browser_factory.create_context(self.browser) + return self.context + + def init_db(self): + self.persistence.create_tables() + return {"status": "ok", "database_url": self.settings.database.url} + + def _warmup_visit(self, page: Page) -> None: + try: + logger.info("Warmup: visiting homepage to pass anti-bot challenge...") + page.goto(self.settings.home_url, wait_until="commit", timeout=30_000) + try: + page.wait_for_load_state("domcontentloaded", timeout=6_000) + except PlaywrightTimeoutError: + pass + try: + page.wait_for_function("() => document.title && document.title.length > 3", timeout=4_000) + except PlaywrightTimeoutError: + pass + time.sleep(0.3) + logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50]) + except Exception as e: + logger.warning("Warmup visit failed: %s — continuing anyway", e) + time.sleep(1.5) + + def _get_page_with_warmup(self) -> Page: + context = self._new_context() + page = context.new_page() + self._warmup_visit(page) + return page + + def _dedupe_urls(self, raw_urls: list[str]) -> list[str]: + vehicle_urls: list[str] = [] + seen: set[str] = set() + for raw in raw_urls: + normalized = self._normalize_vehicle_url(raw) + if normalized not in seen: + seen.add(normalized) + vehicle_urls.append(normalized) + return vehicle_urls + + def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: + url_to_origin_id = { + url: self._extract_db_origin_id_from_url(url) + for url in vehicle_urls + } + candidate_origin_ids = [oid for oid in url_to_origin_id.values() if oid] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( + vehicle_urls, candidate_origin_ids, + ) + known_urls = { + url for url in vehicle_urls + if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) + } + skipped = len(known_urls) + if skipped: + logger.info("Filtering already known vehicles: skipped %d", skipped) + new_urls = [url for url in vehicle_urls if url not in known_urls] + return new_urls, skipped + + def _extract_page_urls( + self, + page_result: ListingPageResult, + all_raw_urls: list[str], + seen_urls: set[str], + all_listing_origin_urls: set[str] | None = None, + ) -> list[str]: + page_urls: list[str] = [] + for item in page_result.vehicle_links: + normalized = self._normalize_vehicle_url(item.href) + all_raw_urls.append(item.href) + if normalized and all_listing_origin_urls is not None: + all_listing_origin_urls.add(normalized) + if normalized and normalized not in seen_urls: + seen_urls.add(normalized) + page_urls.append(normalized) + return page_urls + + @staticmethod + def _build_segment_listing_url(base_url: str, make: str | None) -> str: + """Построить URL листинга для сегмента. Make передаётся через query-параметр.""" + if not make: + return base_url + sep = "&" if "?" in base_url else "?" + return f"{base_url}{sep}Make={make.replace(' ', '%20')}" + + def _recover_empty_listing_page( + self, + page: Page, + *, + page_number: int, + all_raw_urls: list[str], + seen_urls: set[str], + all_listing_origin_urls: set[str] | None = None, + listing_url: str | None = None, + ) -> tuple[ListingPageResult, list[str]]: + if page_number == 1: + logger.warning("Page 1 returned 0 links — retrying listing open...") + time.sleep(3) + self.listing_collector.open_cars_listing(page, url_override=listing_url) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) + + logger.warning( + "Page %d returned 0 links — retrying current page before stopping pagination", + page_number, + ) + try: + page.reload(wait_until="domcontentloaded", timeout=30_000) + except Exception as exc: + logger.debug("Page %d reload failed during empty-page recovery: %s", page_number, exc) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) + + def _open_listing_page( + self, + *, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> tuple[Page, dict[str, str | int | None]]: + page = self._get_page_with_warmup() + try: + self.listing_collector.open_cars_listing(page, url_override=listing_url) + applied_filters = self.listing_collector.apply_filters( + page, + make=make, + model=model, + year_min=year_min, + year_max=year_max, + ) + except Exception: + page.close() + raise + return page, applied_filters + + def _reopen_listing_and_resume( + self, + *, + target_page_number: int, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + max_nav_pages: int = 10, + ) -> tuple[Page, dict[str, str | int | None]]: + # Ограничиваем глубину навигации: если до цели > max_nav_pages кликов — не пытаемся. + if target_page_number > max_nav_pages + 1: + raise ListingResumeError( + f"Cannot resume at page {target_page_number}: " + f"exceeds max navigation depth ({max_nav_pages} pages)" + ) + + self._report_progress( + "listing_resume_started", + target_page=target_page_number, + max_nav_pages=max_nav_pages, + ) + + page, applied_filters = self._open_listing_page( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + self._report_progress( + "listing_resume_opened", + target_page=target_page_number, + ) + + for expected_page in range(2, target_page_number + 1): + self._report_progress( + "listing_resume_progress", + current_page=expected_page - 1, + target_page=target_page_number, + next_page_number=expected_page, + ) + + with _ProgressHeartbeat( + self._report_progress, + "listing_resume_progress", + current_page=expected_page - 1, + target_page=target_page_number, + next_page_number=expected_page, + ): + next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=expected_page) + + if not next_ok: + self._report_progress( + "listing_resume_failed", + current_page=expected_page - 1, + target_page=target_page_number, + ) + page.close() + raise ListingResumeError(f"Failed to resume listing at page {target_page_number}") + + self._report_progress( + "listing_resume_progress", + current_page=expected_page, + target_page=target_page_number, + next_page_number=min(target_page_number, expected_page + 1), + ) + + self._report_progress( + "listing_resume_completed", + current_page=target_page_number, + target_page=target_page_number, + ) + logger.warning("Listing resumed at page %d after recovery", target_page_number) + return page, applied_filters + + def _open_listing_for_stream( + self, + *, + make: str | None, + model: str | None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> tuple[Page, dict[str, str | int | None]]: + # Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие + # сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется + # на уровне сегментов в worker/tasks.py (_save_last_completed_segment). + return self._open_listing_page( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + def collect_listing( + self, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, + ): + try: + max_pages = self.settings.listing.max_pages_per_run + vehicles = list(self.fast_client.iter_listing_vehicles(make=make, max_pages=max_pages)) + vehicle_urls = [f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}" for vehicle in vehicles] + if model: + model_key = model.casefold() + vehicle_urls = [url for url in vehicle_urls if model_key in url.casefold()] + if known_origin_ids: + filtered_urls = [] + for url in vehicle_urls: + origin_id = self._extract_db_origin_id_from_url(url) + if origin_id and origin_id in known_origin_ids: + continue + filtered_urls.append(url) + vehicle_urls = filtered_urls + return { + "status": "ok", + "mode": "fast_hidden_payload", + "vehicle_urls": vehicle_urls, + "vehicles_collected": len(vehicle_urls), + "pages": [], + "early_stopped": False, + "truncated_by_time_budget": False, + } + except Exception as exc: + logger.warning("Fast listing collection failed, falling back to browser listing: %s", exc) + + page = self._get_page_with_warmup() + + try: + listing = self.listing_collector.collect_listing_links( + page, + make=make, + model=model, + known_origin_ids=known_origin_ids, + max_duration_seconds=max_duration_seconds, + ) + finally: + page.close() + + return { + "status": "ok", + **listing, + } + + def _sync_listing_streaming( + self, + *, + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + started_at: float, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, Any]: + batch_size = self.settings.celery.batch_size + pending_urls: list[str] = [] + seen_urls: set[str] = set() + all_raw_urls: list[str] = [] + all_listing_origin_urls: set[str] = set() + pages_info: list[dict[str, Any]] = [] + skipped_existing = 0 + total = 0 + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + early_stopped = False + truncated_by_time_budget = False + listing_recovery_attempts = 0 + + known_origin_ids: set[str] | None = None + threshold = self.settings.listing.early_stop_threshold + if effective_only_new and threshold > 0.0: + try: + known_origin_ids = self.persistence.get_all_origin_ids_for_lane("iaai:") + logger.info( + "Loaded %d known origin_ids for early-stop listing", + len(known_origin_ids), + ) + except Exception as exc: + logger.warning("Could not load known origin_ids for early-stop: %s", exc) + + # Совместимость: если only_new без limit и без сегментного URL — старая схема. + # При сегментированном скрапинге (listing_url/year фильтры) всегда streaming. + if effective_only_new and (limit is None or limit <= 0) and not listing_url and year_min is None and year_max is None: + listing_payload = self.collect_listing( + make=make, + model=model, + known_origin_ids=known_origin_ids, + max_duration_seconds=None, + ) + raw_urls = list(listing_payload.get("vehicle_urls", [])) + deduped_urls = self._dedupe_urls(raw_urls) + fresh_urls, page_skipped = self._filter_known_urls(deduped_urls) + skipped_existing += page_skipped + pending_urls.extend(fresh_urls) + total = len(fresh_urls) + + for raw in raw_urls: + normalized = self._normalize_vehicle_url(raw) + if normalized: + all_listing_origin_urls.add(normalized) + + logger.info( + "Starting sync: %d vehicles to process (batch mode, only_new legacy path)", + total, + ) + + while len(pending_urls) >= batch_size: + batch_urls = pending_urls[:batch_size] + try: + batch_result = self.sync_batch(batch_urls, lane=lane) + cars_upserted += batch_result.get("cars_upserted", 0) + cars_failed += batch_result.get("cars_failed", 0) + protection_events += int(batch_result.get("protection_events", 0)) + images_upserted += batch_result.get("images_upserted", 0) + failures.extend(batch_result.get("failures", [])) + except Exception as batch_exc: + logger.error("Legacy only_new batch failed: %s", batch_exc) + cars_failed += len(batch_urls) + failures.append({"vehicle_url": "legacy_only_new_batch", "error": str(batch_exc)}) + pending_urls = pending_urls[batch_size:] + + if pending_urls: + try: + batch_result = self.sync_batch(pending_urls, lane=lane) + cars_upserted += batch_result.get("cars_upserted", 0) + cars_failed += batch_result.get("cars_failed", 0) + protection_events += int(batch_result.get("protection_events", 0)) + images_upserted += batch_result.get("images_upserted", 0) + failures.extend(batch_result.get("failures", [])) + except Exception as batch_exc: + logger.error("Legacy only_new final batch failed: %s", batch_exc) + cars_failed += len(pending_urls) + failures.append({"vehicle_url": "legacy_only_new_final_batch", "error": str(batch_exc)}) + + return { + "listing": listing_payload, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _process_pending_batch(batch_urls: list[str], batch_start: int) -> bool: + nonlocal cars_upserted, cars_failed, protection_events, images_upserted, failures + if not batch_urls: + return True + + logger.info( + "Processing batch %d-%d of %d", + batch_start + 1, + batch_start + len(batch_urls), + total, + ) + try: + batch_result = self.sync_batch(batch_urls, lane=lane) + cars_upserted += batch_result.get("cars_upserted", 0) + cars_failed += batch_result.get("cars_failed", 0) + protection_events += int(batch_result.get("protection_events", 0)) + images_upserted += batch_result.get("images_upserted", 0) + failures.extend(batch_result.get("failures", [])) + self._report_progress( + "batch_upserted", + cars_upserted=cars_upserted, + cars_failed=cars_failed, + total_discovered=total, + ) + return True + except PlaywrightError as pw_exc: + logger.warning( + "Batch %d-%d browser crashed: %s. Reinitializing browser context.", + batch_start + 1, + batch_start + len(batch_urls), + pw_exc, + ) + cars_failed += len(batch_urls) + failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(pw_exc)}) + try: + self._new_context() + logger.info("Browser context reinitialized successfully after crash") + return True + except Exception as reinit_exc: + logger.error("Failed to reinitialize browser: %s. Aborting remaining batches.", reinit_exc) + return False + except Exception as batch_exc: + logger.error( + "Batch %d-%d failed: %s. Continuing with next batch.", + batch_start + 1, + batch_start + len(batch_urls), + batch_exc, + ) + cars_failed += len(batch_urls) + failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)}) + return True + + def _consume_listing_recovery_attempt(*, page_number: int, reason: str) -> None: + nonlocal listing_recovery_attempts + listing_recovery_attempts += 1 + logger.warning( + "Listing recovery attempt %d/%d at page %d (%s)", + listing_recovery_attempts, + _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT, + page_number, + reason, + ) + if listing_recovery_attempts > _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT: + raise ListingResumeError( + f"Listing recovery exhausted at page {page_number}: {reason}" + ) + + page = None + try: + page, applied_filters = self._open_listing_for_stream( + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + + pages_since_rotation = 0 + for page_number in range(1, self.settings.listing.max_pages_per_run + 1): + # Heartbeat для worker stall-watchdog: при малом числе ссылок на странице + # batch_upserted может долго не вызываться, поэтому пульсуем прогресс + # на каждом шаге пагинации. + self._report_progress( + "listing_page_scan_started", + page_number=page_number, + total_discovered=total, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + + # Проактивная ротация контекста (опционально, по умолчанию выключена). + # Если включена — требует долистывания до page_number после reopen, + # поэтому max_nav_pages поднят под реальную глубину. + if _CONTEXT_ROTATE_EVERY_PAGES > 0 and pages_since_rotation >= _CONTEXT_ROTATE_EVERY_PAGES: + logger.warning( + "Rotating browser context at page %d (every %d pages) to reset anti-bot state", + page_number, _CONTEXT_ROTATE_EVERY_PAGES, + ) + try: + page.close() + except Exception: + pass + page = None + try: + _consume_listing_recovery_attempt( + page_number=page_number, + reason="context_rotation", + ) + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + pages_since_rotation = 0 + except ListingResumeError as resume_exc: + logger.warning( + "Context rotation could not resume at page %d (%s) — stopping segment", + page_number, resume_exc, + ) + break + + try: + with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number}"): + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + except (PageOperationTimeoutError, PlaywrightError) as op_exc: + logger.warning( + "Page %d collect stalled/failed (%s) — reopening listing and resuming", + page_number, op_exc, + ) + try: + page.close() + except Exception: + pass + page = None + try: + _consume_listing_recovery_attempt( + page_number=page_number, + reason=f"collect_failed:{type(op_exc).__name__}", + ) + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + pages_since_rotation = 0 + with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number} (after reopen)"): + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + except (ListingResumeError, PageOperationTimeoutError, PlaywrightError) as resume_exc: + logger.warning( + "Cannot recover at page %d (%s) — stopping pagination for this segment", + page_number, resume_exc, + ) + break + + pages_info.append({ + "page_number": page_result.page_number, + "links_found": len(page_result.vehicle_links), + }) + pages_since_rotation += 1 + + page_urls = self._extract_page_urls( + page_result, + all_raw_urls, + seen_urls, + all_listing_origin_urls, + ) + + if not page_urls: + page_result, page_urls = self._recover_empty_listing_page( + page, + page_number=page_number, + all_raw_urls=all_raw_urls, + seen_urls=seen_urls, + all_listing_origin_urls=all_listing_origin_urls, + listing_url=listing_url, + ) + if not page_urls and page_number > 1: + logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number) + page.close() + try: + _consume_listing_recovery_attempt( + page_number=page_number, + reason="empty_page_after_retry", + ) + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + page_urls = self._extract_page_urls( + page_result, + all_raw_urls, + seen_urls, + all_listing_origin_urls, + ) + except ListingResumeError as resume_exc: + logger.warning( + "Cannot resume at page %d (%s) — stopping pagination for this segment", + page_number, resume_exc, + ) + page = None + page_urls = [] + if not page_urls: + logger.info("Page %d: 0 new links, stopping pagination", page_number) + break + + if known_origin_ids is not None and threshold > 0.0 and page_result.vehicle_links: + page_known = sum( + 1 + for item in page_result.vehicle_links + if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids + ) + page_new = len(page_result.vehicle_links) - page_known + ratio = page_known / len(page_result.vehicle_links) + if ratio >= threshold and page_new == 0: + logger.info( + "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", + page_number, + ratio * 100, + page_known, + len(page_result.vehicle_links), + threshold * 100, + ) + early_stopped = True + break + + fresh_urls = page_urls + page_skipped = 0 + if effective_only_new: + fresh_urls, page_skipped = self._filter_known_urls(page_urls) + skipped_existing += page_skipped + + if limit is not None and limit > 0: + remaining_limit = max(0, limit - total - len(pending_urls)) + if remaining_limit <= 0: + break + fresh_urls = fresh_urls[:remaining_limit] + + pending_urls.extend(fresh_urls) + total += len(fresh_urls) + + logger.info( + "Page %d: %d links, %d new, %d known (total new: %d/%s)", + page_number, + len(page_urls), + len(fresh_urls), + page_skipped, + total, + limit if limit is not None else "∞", + ) + self._report_progress( + "listing_page_scan_done", + page_number=page_number, + page_links=len(page_urls), + page_new=len(fresh_urls), + page_known=page_skipped, + total_discovered=total, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + + # Прогресс каждые 10 страниц на уровне WARNING. + if page_number % 10 == 0: + logger.warning( + "sync_listing progress: pages=%d, discovered=%d, upserted=%d, failed=%d, pending=%d", + page_number, + total, + cars_upserted, + cars_failed, + len(pending_urls), + ) + + while len(pending_urls) >= batch_size: + batch_urls = pending_urls[:batch_size] + self._report_progress( + "listing_batch_dispatch_started", + page_number=page_number, + batch_size=len(batch_urls), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + if not _process_pending_batch(batch_urls, cars_upserted + cars_failed): + pending_urls = [] + break + pending_urls = pending_urls[batch_size:] + self._report_progress( + "listing_batch_dispatch_done", + page_number=page_number, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + # Пауза между батчами: снижает нагрузку на IAAI и риск бана. + if pending_urls: + time.sleep(random.uniform(0.5, 1.5)) + + # Anti-stall: не держим хвост pending до следующей страницы/конца сегмента. + # Если после scan остались URL меньше batch_size — отправляем их сразу. + if pending_urls: + self._report_progress( + "listing_tail_batch_started", + page_number=page_number, + batch_size=len(pending_urls), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + if not _process_pending_batch(pending_urls, cars_upserted + cars_failed): + pending_urls = [] + break + pending_urls = [] + self._report_progress( + "listing_tail_batch_done", + page_number=page_number, + pending_urls=0, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + + if limit is not None and limit > 0 and total >= limit: + break + if ( + self.settings.listing.collect_current_page_only + or not self.settings.listing.include_pagination + or not page_result.next_page_detected + ): + break + + if page_number == 1 and not self.listing_collector.has_page_number(page, 2): + logger.info( + "Page 2 not found on page 1 — treating segment as single-page", + ) + self._report_progress( + "listing_single_page_no_page2", + page_number=page_number, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + break + + suspicious_small_segment = ( + total <= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS + and page_number >= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE + ) + self._report_progress( + "listing_next_page_started", + page_number=page_number, + next_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + try: + with _ProgressHeartbeat( + self._report_progress, + "listing_next_page_started", + page_number=page_number, + next_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ): + with _PageOpWatchdog(_PAGE_NEXT_TIMEOUT_S, f"next page {page_number + 1}"): + next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1) + except (PageOperationTimeoutError, PlaywrightError) as nav_exc: + logger.warning( + "go_to_next_page stalled/failed at page %d (%s) — will reopen", + page_number + 1, nav_exc, + ) + next_ok = False + self._report_progress( + "listing_next_page_done", + page_number=page_number, + next_page_number=page_number + 1, + next_ok=bool(next_ok), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + if not next_ok: + if page_number == 1: + logger.warning( + "Page 2 is not reachable from page 1 — treating segment as single-page and moving on", + ) + break + if suspicious_small_segment: + logger.warning( + "Small segment pagination looks exhausted at page %d: total=%d, next page navigation failed — stopping segment", + page_number, + total, + ) + break + logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1) + self._report_progress( + "listing_reopen_started", + page_number=page_number, + target_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + try: + page.close() + except Exception: + pass + page = None + try: + _consume_listing_recovery_attempt( + page_number=page_number + 1, + reason=f"next_page_failed:{type(nav_exc).__name__}", + ) + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number + 1, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + pages_since_rotation = 0 + self._report_progress( + "listing_reopen_done", + page_number=page_number, + target_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + except ListingResumeError as resume_exc: + logger.warning( + "Cannot resume at page %d (%s) — treating pagination as exhausted", + page_number + 1, resume_exc, + ) + self._report_progress( + "listing_reopen_failed", + page_number=page_number, + target_page_number=page_number + 1, + error=str(resume_exc), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + break + + if pending_urls: + _process_pending_batch(pending_urls, cars_upserted + cars_failed) + + logger.warning( + "sync_listing final progress: pages=%d, discovered=%d, upserted=%d, failed=%d", + len(pages_info), + total, + cars_upserted, + cars_failed, + ) + finally: + if page is not None: + page.close() + + return { + "listing": { + "status": "ok", + "listing_url": self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": len(pages_info), + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": early_stopped, + "truncated_by_time_budget": truncated_by_time_budget, + "pages": pages_info, + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _get_page(self) -> Page: + if not self.context: + self._new_context() + return self.context.new_page() + + @retryable(max_attempts=3, jitter_seconds=0.25) + def scrape_vehicle_detail(self, vehicle_url: str): + try: + return self._scrape_vehicle_detail_fast(vehicle_url) + except Exception as exc: + logger.warning("Fast detail scrape failed for %s, falling back to browser: %s", vehicle_url, exc) + + page = self._get_page() + try: + return self._scrape_on_page(page, vehicle_url) + finally: + page.close() + + def _scrape_vehicle_detail_fast(self, vehicle_url: str) -> dict[str, Any]: + trace_id = self._new_trace_id("scrape-fast") + started_at = time.perf_counter() + origin_id = self._extract_origin_id_from_url(vehicle_url) + if not origin_id: + raise RuntimeError(f"Could not extract IAAI inventory id from URL: {vehicle_url}") + + detail_payload = self.fast_client.fetch_vehicle_detail_payload(origin_id) + db_record = self.fast_mapper.map_payload_to_record( + detail_payload=detail_payload, + vehicle_url=self._normalize_vehicle_url(vehicle_url), + ) + vehicle_summary = self.fast_mapper.payload_to_summary(detail_payload, vehicle_url) + if not (vehicle_summary.get("make") or vehicle_summary.get("lot_number")): + raise SiteStructureChangedError(f"ProductDetailsVM returned no vehicle identity for {vehicle_url}") + + return { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": {"requests": [], "json_responses": [], "capture_limits": {}}, + "vehicle_summary": vehicle_summary, + "payload_insights": {"source": "ProductDetailsVM"}, + "embedded_json": [detail_payload], + "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {"mode": "fast_hidden_payload"}, + "db_record": db_record.model_dump(mode="json"), + } + + _JS_EXTRACT = """ + () => { + try { + const scripts = document.querySelectorAll('script:not([src])'); + for (const s of scripts) { + const t = s.textContent || ''; + if (!t.includes('inventoryView') || !t.includes('attributes')) continue; + const start = t.indexOf('{'); + if (start < 0) continue; + let depth = 0, end = -1; + for (let i = start; i < t.length; i++) { + if (t[i] === '{') depth++; + else if (t[i] === '}') { depth--; if (depth === 0) { end = i; break; } } + } + if (end < 0) continue; + try { + const obj = JSON.parse(t.slice(start, end + 1)); + const iv = obj.inventoryView; + if (!iv || !iv.attributes) continue; + const attr = iv.attributes; + const imgs = (iv.imageDimensions && iv.imageDimensions.keys && iv.imageDimensions.keys.$values) + ? iv.imageDimensions.keys.$values : []; + const bid = (obj.auctionInformation && obj.auctionInformation.biddingInformation) + ? obj.auctionInformation.biddingInformation : {}; + const prebid = (obj.auctionInformation && obj.auctionInformation.prebidInformation) + ? obj.auctionInformation.prebidInformation : {}; + return { + ok: true, + SalvageId: attr.SalvageId || '', + StockNumber: attr.StockNumber || '', + Year: attr.Year || '', + Make: attr.Make || '', + Model: attr.Model || '', + Series: attr.Series || '', + BodyStyleName: attr.BodyStyleName || '', + Cylinders: attr.Cylinders || '', + DriveLineTypeDesc: attr.DriveLineTypeDesc || '', + EngineSize: (attr.EngineInformation || attr.EngineSize || '').trim(), + FuelTypeCode: attr.FuelTypeCode || '', + Transmission: attr.Transmission || '', + ExteriorColor: attr.ExteriorColor || '', + PrimaryDamageDesc: attr.PrimaryDamageDesc || '', + SecondaryDamageDesc: attr.SecondaryDamageDesc || '', + ODOValue: attr.ODOValue || '', + ODOBrand: attr.ODOBrand || '', + RunAndDrive: attr.RunAndDrive || '', + Keys: attr.Keys || '', + BranchName: attr.BranchName || '', + AuctionDateTime: attr.AuctionDateTime || '', + Title: attr.Title || '', + TitleBrand: attr.TitleBrand || '', + TitleCode: attr.TitleCode || '', + EstRepairCost: attr.EstRepairCost || '', + VehicleGrade: attr.VehicleGrade || '', + highBidAmount: prebid.highBidAmount || bid.highBidAmount || '', + buyNowPrice: prebid.buyNowPrice || bid.buyNowPrice || '', + acv: attr.ProviderACV || '', + BranchNumber: attr.BranchNumber || '', + imageKeys: imgs.map(i => i.k || '').filter(Boolean), + }; + } catch (_) { continue; } + } + } catch (_) {} + return { ok: false }; + } + """ + + @staticmethod + def _build_car_from_js(js_data: dict, vehicle_url: str) -> dict: + if not js_data or not js_data.get("ok"): + return {} + + brnch = str(js_data.get("BranchNumber", "") or "").strip() + img_keys = js_data.get("imageKeys") or [] + image_urls = [ + f"https://vis.iaai.com/resizer?imageKeys={k}&width=845&height=633" + for k in img_keys + ] + + return { + "source_url": vehicle_url, + "lot_number": js_data.get("StockNumber") or js_data.get("SalvageId"), + "year": js_data.get("Year"), + "make": js_data.get("Make"), + "model": js_data.get("Model"), + "trim": js_data.get("Series"), + "body_type": js_data.get("BodyStyleName"), + "cylinders": js_data.get("Cylinders"), + "drive": js_data.get("DriveLineTypeDesc"), + "engine": js_data.get("EngineSize"), + "fuel_type": js_data.get("FuelTypeCode"), + "gearbox": js_data.get("Transmission"), + "color": js_data.get("ExteriorColor"), + "primary_damage": js_data.get("PrimaryDamageDesc"), + "secondary_damage": js_data.get("SecondaryDamageDesc"), + "odometer": js_data.get("ODOValue"), + "run_and_drive": js_data.get("RunAndDrive"), + "keys": js_data.get("Keys"), + "location": js_data.get("BranchName"), + "auction_date": js_data.get("AuctionDateTime"), + "title": js_data.get("Title"), + "current_bid": js_data.get("highBidAmount"), + "buy_now": js_data.get("buyNowPrice"), + "actual_cash_value": js_data.get("acv"), + "estimated_repair_cost": js_data.get("EstRepairCost"), + "image_urls": image_urls, + } + + @staticmethod + def _build_payload_insights(vehicle_summary: dict) -> dict: + return { + "vehicle_core": vehicle_summary, + "pricing": { + "buy_now": vehicle_summary.get("buy_now"), + "current_bid": vehicle_summary.get("current_bid"), + "actual_cash_value": vehicle_summary.get("actual_cash_value"), + "estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"), + "currency": "USD", + }, + "bids": {"amount": vehicle_summary.get("current_bid"), "currency": "USD"}, + "damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")}, + "auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")}, + "images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []}, + } + + def _scrape_on_page(self, page: Page, vehicle_url: str): + trace_id = self._new_trace_id("scrape") + started_at = time.perf_counter() + + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + + capture = NetworkCapture(self.settings) + capture.attach(page, origin_url=vehicle_url) + + page.goto(vehicle_url, wait_until="commit", timeout=60_000) + + try: + page.wait_for_load_state("domcontentloaded", timeout=5_000) + except PlaywrightTimeoutError: + pass + + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError(f"JS extraction returned no vehicle identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + network_dump = capture.export() + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + "vehicle_summary": vehicle_summary, + "payload_insights": {}, + "embedded_json": [], + "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {}, + "db_record": db_record.model_dump(mode="json"), + } + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + # Резервный путь: полный HTML-парсинг. + try: + page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=400) + except PlaywrightTimeoutError: + pass + + html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = "" + network_dump = capture.export() + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, vehicle_url) + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + **parsed, + "db_record": db_record.model_dump(mode="json"), + } + + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + @staticmethod + def _extract_iaai_json_from_html(html: str, vehicle_url: str) -> dict | None: + """Извлекает IAAI inventoryView.attributes из HTML без браузера. + + Использует json.JSONDecoder.raw_decode для быстрого поиска JSON + вместо посимвольного сканирования скобок. + """ + try: + payload = parse_product_details_vm(html) + db_record = IAAIScraper._fast_payload_to_js_data(payload) + if db_record: + return db_record + except Exception: + pass + + search = "inventoryView" + pos = html.find(search) + if pos < 0: + return None + + script_start = html.rfind("", script_start) + if tag_end < 0: + return None + content_start = html.find("{", tag_end) + if content_start < 0: + return None + + decoder = json.JSONDecoder() + try: + obj, _ = decoder.raw_decode(html, content_start) + except (json.JSONDecodeError, ValueError): + return None + + iv = obj.get("inventoryView") + if not iv or not iv.get("attributes"): + return None + + attr = iv["attributes"] + imgs = [] + try: + imgs = iv.get("imageDimensions", {}).get("keys", {}).get("$values", []) or [] + except Exception: + pass + + bid = {} + prebid = {} + try: + ai = obj.get("auctionInformation", {}) + bid = ai.get("biddingInformation", {}) or {} + prebid = ai.get("prebidInformation", {}) or {} + except Exception: + pass + + img_keys = [i.get("k", "") for i in imgs if i.get("k")] + return { + "ok": True, + "SalvageId": attr.get("SalvageId", ""), + "StockNumber": attr.get("StockNumber", ""), + "Year": attr.get("Year", ""), + "Make": attr.get("Make", ""), + "Model": attr.get("Model", ""), + "Series": attr.get("Series", ""), + "BodyStyleName": attr.get("BodyStyleName", ""), + "Cylinders": attr.get("Cylinders", ""), + "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), + "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), + "FuelTypeCode": attr.get("FuelTypeCode", ""), + "Transmission": attr.get("Transmission", ""), + "ExteriorColor": attr.get("ExteriorColor", ""), + "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), + "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), + "ODOValue": attr.get("ODOValue", ""), + "ODOBrand": attr.get("ODOBrand", ""), + "RunAndDrive": attr.get("RunAndDrive", ""), + "Keys": attr.get("Keys", ""), + "BranchName": attr.get("BranchName", ""), + "AuctionDateTime": attr.get("AuctionDateTime", ""), + "Title": attr.get("Title", ""), + "TitleBrand": attr.get("TitleBrand", ""), + "TitleCode": attr.get("TitleCode", ""), + "EstRepairCost": attr.get("EstRepairCost", ""), + "VehicleGrade": attr.get("VehicleGrade", ""), + "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), + "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), + "acv": attr.get("ProviderACV", ""), + "BranchNumber": attr.get("BranchNumber", ""), + "imageKeys": img_keys, + } + + @staticmethod + def _fast_payload_to_js_data(payload: dict[str, Any]) -> dict[str, Any] | None: + inventory_view = payload.get("inventoryView") if isinstance(payload, dict) else None + if not isinstance(inventory_view, dict): + return None + attr = inventory_view.get("attributes") + if not isinstance(attr, dict): + return None + image_dimensions = inventory_view.get("imageDimensions") + image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {} + keys_container = image_dimensions.get("keys") + keys_container = keys_container if isinstance(keys_container, dict) else {} + image_keys = keys_container.get("$values") + image_keys = image_keys if isinstance(image_keys, list) else [] + auction_info = payload.get("auctionInformation") + auction_info = auction_info if isinstance(auction_info, dict) else {} + bid = auction_info.get("biddingInformation") + bid = bid if isinstance(bid, dict) else {} + prebid = auction_info.get("prebidInformation") + prebid = prebid if isinstance(prebid, dict) else {} + return { + "ok": True, + "SalvageId": attr.get("SalvageId", ""), + "StockNumber": attr.get("Id") or attr.get("StockNumber", ""), + "Year": attr.get("Year", ""), + "Make": attr.get("Make", ""), + "Model": attr.get("Model", ""), + "Series": attr.get("Series", ""), + "BodyStyleName": attr.get("BodyStyleName", ""), + "Cylinders": attr.get("Cylinders", ""), + "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), + "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), + "FuelTypeCode": attr.get("FuelTypeCode", ""), + "Transmission": attr.get("Transmission", ""), + "ExteriorColor": attr.get("ExteriorColor", ""), + "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), + "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), + "ODOValue": attr.get("ODOValue", ""), + "ODOBrand": attr.get("ODOBrand", ""), + "RunAndDrive": attr.get("RunAndDrive", ""), + "Keys": attr.get("Keys", ""), + "BranchName": attr.get("BranchName", ""), + "AuctionDateTime": attr.get("AuctionDateTime", ""), + "Title": attr.get("Title", ""), + "TitleBrand": attr.get("TitleBrand", ""), + "TitleCode": attr.get("TitleCode", ""), + "EstRepairCost": attr.get("EstRepairCost", ""), + "VehicleGrade": attr.get("VehicleGrade", ""), + "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), + "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), + "acv": attr.get("ProviderACV", ""), + "BranchNumber": attr.get("BranchNumber", ""), + "imageKeys": [item.get("k", "") for item in image_keys if isinstance(item, dict) and item.get("k")], + } + + def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord: + if not self.context: + self._new_context() + assert self.context is not None + last_error: Exception | None = None + max_attempts = max(1, self.settings.fast_path_max_attempts) + timeout_ms = max(1000, self.settings.fast_path_timeout_ms) + for attempt in range(1, max_attempts + 1): + try: + response = self.context.request.get(vehicle_url, timeout=timeout_ms) + if not response.ok: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + + html = response.text() + + js_data = self._extract_iaai_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + except Exception as exc: + last_error = exc + if attempt < max_attempts: + time.sleep(0.2) + + if last_error is not None: + raise last_error + raise RuntimeError(f"HTTP fast-path failed for {vehicle_url}") + + def _cookie_header_for_context(self) -> str: + if not self.context: + return "" + try: + cookies = self.context.cookies() + except Exception: + return "" + pairs = [ + f"{item.get('name')}={item.get('value')}" + for item in cookies + if item.get("name") and item.get("value") is not None + ] + return "; ".join(pairs) + + def _scrape_via_raw_http( + self, + vehicle_url: str, + *, + cookie_header: str, + user_agent: str, + ) -> CarRecord: + """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive).""" + headers = { + "User-Agent": user_agent, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + } + if cookie_header: + headers["Cookie"] = cookie_header + + response = self._http_pool.request("GET", vehicle_url, headers=headers) + if response.status >= 400: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + html = response.data.decode("utf-8", errors="ignore") + if is_challenge_response(status_code=int(response.status), body_text=html, expected_marker=DETAIL_MARKER): + raise AntiBotDetectedError(f"IAAI challenge detected for {vehicle_url}") + + js_data = self._extract_iaai_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + + def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"): + trace_id = self._new_trace_id("sync-vehicle") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + ids_fetched = 1 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + status = "failed" + error_summary = None + try: + scrape_result = self.scrape_vehicle_detail(vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + cars_upserted = 1 + images_upserted = int(upsert.get("images_upserted", 0)) + status = "success" + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + "vehicle_url": vehicle_url, + "db_action": upsert.get("action"), + "images_upserted": images_upserted, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "db_record": db_record, + } + except Exception as exc: + cars_failed = 1 + status = "failed" + error_summary = str(exc) + raise + finally: + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=ids_fetched, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + # ── Настройки sync_batch ── + _HTTP_MICRO_BATCH = 25 # URL за микро-батч + _HTTP_MAX_RETRIES = 2 # Повторов на URL до перехода в браузер + _HTTP_RETRY_DELAYS = (0.4, 1.0) # Задержки между повторами + _FALLBACK_PARALLEL_PAGES = 4 # Параллельных вкладок для fallback + + def _browser_fallback_parallel( + self, + fallback_urls: list[tuple[str, int]], + total: int, + n_pages: int, + ) -> dict: + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + cars_failed = 0 + protection_events = 0 + + slices: list[list[tuple[str, int]]] = [[] for _ in range(n_pages)] + for i, item in enumerate(fallback_urls): + slices[i % n_pages].append(item) + + def _process_slice(url_slice: list[tuple[str, int]]) -> dict: + local_records: list[CarRecord] = [] + local_failures: list[dict[str, str]] = [] + local_failed = 0 + local_protection = 0 + page: Page | None = None + processed_local = 0 + try: + for url, global_idx in url_slice: + processed_local += 1 + # Heartbeat в Redis progress: даже если anti-bot тормозит fallback, + # watchdog видит живую задачу и не убивает её как stalled. + self._report_progress( + "browser_fallback_progress", + fallback_processed=processed_local, + fallback_total=len(url_slice), + vehicle_index=global_idx, + vehicle_total=total, + ) + + # Если anti-bot уже активен (много fallback URL), снижаем burst-нагрузку. + if len(url_slice) >= 20: + time.sleep(random.uniform(0.2, 0.7)) + + if page is None: + page = self._get_page() + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + + try: + page.goto( + url, + wait_until="commit", + timeout=max(3_000, int(self.settings.fallback_navigation_timeout_ms)), + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed to open %s: %s", global_idx, total, url, exc) + try: + page.close() + except Exception: + pass + page = None + continue + + try: + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if not js_data.get("ok"): + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except PlaywrightTimeoutError: + pass + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + vehicle_summary = self._build_car_from_js(js_data, url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError( + f"JS extraction returned no vehicle identity for {url}" + ) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + else: + try: + page.wait_for_selector( + "#VehicleDetailViewModel, .veh-details, .vehicle-details, " + "[data-uname='vehicleDetailPage']", + timeout=400, + ) + except PlaywrightTimeoutError: + pass + page_html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = "" + network_dump = { + "requests": [], + "json_responses": [], + "capture_limits": {}, + } + parsed = self.vehicle_parser.normalize(url, page_html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, url) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + record = CarRecord.model_validate(db_record.model_dump(mode="json")) + local_records.append(record) + logger.debug( + "[%d/%d] Parsed %s %s %s (fallback)", + global_idx, total, record.brand, record.model, record.year or "?", + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) + finally: + if page is not None: + try: + page.close() + except Exception: + pass + return { + "records": local_records, + "failures": local_failures, + "cars_failed": local_failed, + "protection_events": local_protection, + } + + # Одна страница — в главном потоке. + if n_pages == 1: + res = _process_slice(slices[0] if slices else []) + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + else: + # Несколько страниц — параллельно, каждый поток со своей Page. + # Таймаут 5 минут на весь fallback — если страницы зависли, не блокируем навсегда. + _fallback_timeout = max(300, len(fallback_urls) * 30) + with ThreadPoolExecutor(max_workers=n_pages) as executor: + futures = [executor.submit(_process_slice, s) for s in slices if s] + for fut in as_completed(futures, timeout=_fallback_timeout): + try: + res = fut.result(timeout=60) + except FuturesTimeoutError: + logger.error("Browser fallback thread timed out") + cars_failed += 1 + continue + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + + return { + "records": records, + "failures": failures, + "cars_failed": cars_failed, + "protection_events": protection_events, + } + + def sync_batch( + self, + vehicle_urls: list[str], + lane: str = "iaai_cars", + parallel_tabs: int | None = None, + ) -> dict: + # Runtime config может меняться на лету (добавили/убрали бренды, диапазоны и т.п.). + # Подхватываем обновление перед каждым batch, чтобы фильтрация применялась сразу. + self._reload_runtime_config() + + trace_id = self._new_trace_id("sync-batch") + started_at = time.perf_counter() + num_workers = parallel_tabs or self.settings.parallel_tabs + num_workers = min(num_workers, len(vehicle_urls), 48) + + cars_upserted = 0 + cars_failed = 0 + cars_filtered = 0 + images_upserted = 0 + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + http_successes = 0 + http_fallbacks = 0 + protection_events = 0 + + total = len(vehicle_urls) + logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) + self._report_progress( + "sync_batch_started", + batch_total=total, + batch_workers=num_workers, + http_ok=0, + http_fallback=0, + batch_records_ready=0, + batch_failures=0, + ) + + cookie_header = self._cookie_header_for_context() + user_agent = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + + # ── Фаза 1: HTTP fast-path ── + fallback_urls: list[tuple[str, int]] = [] + processed_indices: set[int] = set() + fallback_indices: set[int] = set() + + def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: + idx, url = idx_url + # Небольшой анти-бёрст джиттер: разносим старт запросов в пуле, + # чтобы N воркеров не били в одну TLS-/PX-волну. + time.sleep(random.uniform(0.0, 0.15)) + last_exc: Exception | None = None + for attempt in range(1 + self._HTTP_MAX_RETRIES): + try: + return idx, self._scrape_via_raw_http( + url, + cookie_header=cookie_header, + user_agent=user_agent, + ) + except Exception as exc: + last_exc = exc + if attempt < self._HTTP_MAX_RETRIES: + time.sleep(self._HTTP_RETRY_DELAYS[min(attempt, len(self._HTTP_RETRY_DELAYS) - 1)]) + return idx, last_exc # type: ignore[return-value] + + indexed_urls = list(enumerate(vehicle_urls)) + # Таймаут на весь HTTP-пул: (connect 5 + read 20) × retries × URLs / workers + запас. + # Если пул зависнет дольше — отпускаем зависшие потоки и уходим в fallback. + _per_url_budget = (5 + 20) * (1 + self._HTTP_MAX_RETRIES) + 5 + _http_phase_timeout = max(120, _per_url_budget * max(1, total // max(1, num_workers))) + http_processed = 0 + with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor: + try: + for idx, result in executor.map( + _fetch_one_with_retry, indexed_urls, timeout=_http_phase_timeout, + ): + processed_indices.add(idx) + http_processed += 1 + if isinstance(result, Exception): + http_fallbacks += 1 + logger.debug( + "[%d/%d] HTTP fast-path failed for %s: %s", + idx + 1, total, vehicle_urls[idx], result, + ) + fallback_urls.append((vehicle_urls[idx], idx + 1)) + fallback_indices.add(idx) + else: + records.append(result) + http_successes += 1 + logger.debug( + "[%d/%d] Parsed %s %s %s (raw HTTP)", + idx + 1, total, result.brand, result.model, result.year or "?", + ) + self._report_progress( + "sync_batch_http_progress", + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + except FuturesTimeoutError: + logger.error( + "HTTP phase timed out after %ds; %d/%d processed, rest go to fallback", + _http_phase_timeout, http_successes + http_fallbacks, total, + ) + # Все ещё не обработанные URL → в fallback + for i, url in enumerate(vehicle_urls): + if i in processed_indices or i in fallback_indices: + continue + if (url, i + 1) not in fallback_urls: + fallback_urls.append((url, i + 1)) + fallback_indices.add(i) + http_fallbacks += 1 + + logger.info( + "HTTP phase done: %d OK, %d fallback (%.1fs)", + http_successes, http_fallbacks, time.perf_counter() - started_at, + ) + self._report_progress( + "sync_batch_http_done", + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + + # ── Фаза 2: браузерный fallback ── + if fallback_urls and self.settings.scraping_profile.browser_fallback_enabled: + logger.info( + "Fallback browser mode: %d/%d URLs, single page", + len(fallback_urls), total, + ) + self._report_progress( + "sync_batch_fallback_started", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + ) + fb_results = self._browser_fallback_parallel(fallback_urls, total, 1) + records.extend(fb_results["records"]) + cars_failed += fb_results["cars_failed"] + protection_events += fb_results["protection_events"] + failures.extend(fb_results["failures"]) + self._report_progress( + "sync_batch_fallback_done", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + protection_events=protection_events, + ) + elif fallback_urls: + cars_failed += len(fallback_urls) + failures.extend({"vehicle_url": url, "error": "HTTP fast-path failed; browser fallback disabled"} for url, _ in fallback_urls) + logger.warning("Browser fallback disabled by profile; %d URLs marked failed", len(fallback_urls)) + + # ── Фаза 2.5: пост-фильтр по runtime_config ── + filters_cfg = self.runtime_config.filters + if records and not filters_cfg.is_empty(): + pre_filter_count = len(records) + records = [ + rec for rec in records + if filters_cfg.matches({ + "brand": rec.brand, + "model": rec.model, + "year": rec.year, + "body_type": rec.body_type, + "color": rec.color, + "drive": rec.drive, + "gearbox": rec.gearbox, + "price": rec.price, + "mileage": rec.mileage, + }) + ] + cars_filtered = pre_filter_count - len(records) + if cars_filtered: + logger.info( + "Runtime filter: %d/%d records filtered out before DB upsert", + cars_filtered, pre_filter_count, + ) + + # ── Фаза 3: запись в БД ── + if records: + seen_origins: set[str] = set() + unique_records: list[CarRecord] = [] + for rec in records: + key = rec.origin_id or rec.origin_url + if key not in seen_origins: + seen_origins.add(key) + unique_records.append(rec) + if len(unique_records) < len(records): + logger.info("Dedup before DB: %d → %d", len(records), len(unique_records)) + records = unique_records + + try: + self._report_progress( + "sync_batch_db_upsert_started", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) + _db_start = time.perf_counter() + batch_result = self.persistence.upsert_cars_batch(records) + _db_elapsed = time.perf_counter() - _db_start + cars_upserted = batch_result["inserted"] + batch_result["updated"] + images_upserted = batch_result["images_upserted"] + if _db_elapsed > 10: + logger.warning("DB upsert slow: %.1fs for %d records", _db_elapsed, len(records)) + self._report_progress( + "sync_batch_db_upsert_done", + batch_total=total, + batch_records_ready=len(records), + cars_upserted=cars_upserted, + images_upserted=images_upserted, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) + except Exception as exc: + logger.error("Batch upsert failed: %s", exc) + cars_failed += len(records) + failures.append({"vehicle_url": "db_batch", "error": str(exc)}) + self._report_progress( + "sync_batch_db_upsert_failed", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + error=str(exc), + ) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + self._report_progress( + "sync_batch_done", + batch_total=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + cars_filtered=cars_filtered, + images_upserted=images_upserted, + http_ok=http_successes, + http_fallback=http_fallbacks, + protection_events=protection_events, + batch_failures=len(failures), + ) + return { + "trace_id": trace_id, + "status": status, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "cars_filtered": cars_filtered, + "images_upserted": images_upserted, + "http_successes": http_successes, + "http_fallbacks": http_fallbacks, + "protection_events": protection_events, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + def sync_listing( + self, + make: str | None = None, + model: str | None = None, + lane: str = "iaai_cars", + limit: int | None = None, + only_new: bool | None = None, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + skip_mark_sold: bool = False, + ): + # Подхватываем актуальный runtime_config на каждый запуск sync, + # чтобы добавленные/удалённые бренды/модели применялись без рестарта. + self._reload_runtime_config() + + # runtime_config — дефолты; CLI/API аргументы приоритетнее. + rc = self.runtime_config.sync + if limit is None and rc.limit is not None: + limit = rc.limit + if only_new is None and rc.only_new is not None: + only_new = rc.only_new + if lane == "iaai_cars" and rc.lane is not None: + lane = rc.lane + + trace_id = self._new_trace_id("sync-listing") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + cars_upserted = 0 + cars_failed = 0 + cars_filtered = 0 + protection_events = 0 + images_upserted = 0 + total = 0 + skipped_existing = 0 + is_partial_scan = True + failures: list[dict[str, str]] = [] + listing: dict = {} + stream_result: dict[str, Any] = {} + + try: + effective_only_new = self.settings.sync_only_new if only_new is None else only_new + if self.settings.scraping_profile.http_first: + if year_min is not None or year_max is not None: + logger.warning( + "Fast HTTP-first profile ignores year split %s-%s and uses iaai-fast hidden-payload flow", + year_min, + year_max, + ) + fast_engine = FastSyncEngine( + client=self.fast_client, + mapper=self.fast_mapper, + persistence=self.persistence, + batch_size=self.settings.celery.batch_size, + fetch_concurrency=self.settings.fetch_concurrency, + report_progress=self._report_progress, + ) + stream_result = fast_engine.sync_listing( + runtime_config=self.runtime_config, + make=make, + model=model, + lane=lane, + limit=limit, + only_new=effective_only_new, + listing_url=listing_url, + max_pages=self.settings.listing.max_pages_per_run, + skip_mark_sold=skip_mark_sold, + ) + else: + # Stable profile keeps the legacy browser streaming path. + stream_result = self._sync_listing_streaming( + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + started_at=started_at, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + listing = stream_result["listing"] + total = stream_result["total"] + skipped_existing = stream_result["skipped_existing"] + cars_upserted = stream_result["cars_upserted"] + cars_failed = stream_result["cars_failed"] + cars_filtered = int(stream_result.get("cars_filtered", 0)) + images_upserted = stream_result["images_upserted"] + protection_events = int(stream_result.get("protection_events", 0)) + failures.extend(stream_result["failures"]) + all_listing_origin_urls = stream_result["all_listing_origin_urls"] + + logger.info("Streaming sync processed %d vehicles", total) + + # Помечаем проданные авто, исчезнувшие из листинга (только при полном скане). + is_partial_scan = ( + (limit is not None and limit > 0) + or make is not None + or model is not None + or listing_url is not None + or listing.get("early_stopped", False) + or listing.get("truncated_by_time_budget", False) + or bool(stream_result.get("full_scan_completed") is False) + ) + if skip_mark_sold: + logger.debug("Skipping mark_sold: caller requested") + elif all_listing_origin_urls and not is_partial_scan: + try: + sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) + if sold_count: + logger.info("Marked %d cars as sold", sold_count) + except Exception as exc: + logger.warning("Failed to mark sold cars: %s", exc) + elif is_partial_scan: + logger.debug("Skipping mark_sold: partial scan (only_new=%s, limit=%s, make=%s, model=%s, listing_url=%s, early_stopped=%s)", + effective_only_new, limit, make, model, listing_url, listing.get("early_stopped", False)) + except Exception as exc: + if not failures: + failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) + logger.error("sync_listing failed: %s (partial progress: %d upserted)", exc, cars_upserted) + finally: + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + logger.info( + "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images", + run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted, + ) + discovered_total = max(0, int(total)) + fail_ratio = (cars_failed / discovered_total) if discovered_total > 0 else 0.0 + protection_ratio = (protection_events / discovered_total) if discovered_total > 0 else 0.0 + anti_bot_detected = discovered_total > 0 and ( + (protection_events >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + # partial_success допустим — отдельные машины могли не спарситься, + # это не повод повторять весь bootstrap. + "full_scan_completed": bool(stream_result.get("full_scan_completed", (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected)), + "only_new_effective": effective_only_new, + "listing": listing, + "total_discovered": discovered_total, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "cars_filtered": cars_filtered, + "images_upserted": images_upserted, + "protection_events": protection_events, + "anti_bot_detected": anti_bot_detected, + "fail_ratio": round(fail_ratio, 4), + "protection_ratio": round(protection_ratio, 4), + "skipped_existing": skipped_existing, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + def sync_listing_segmented( + self, + segments: list[dict[str, Any]], + lane: str = "iaai_cars", + only_new: bool | None = None, + start_segment: int = 0, + start_page: int = 1, + progress_callback: Callable[[int], None] | None = None, + ) -> dict[str, Any]: + """Итеративный sync_listing по списку сегментов (бренд / бренд+годы). + + Args: + segments: список dict с ключами make, year_min, year_max. + start_segment: индекс сегмента для resume (0-based). + start_page: не используется (остаётся для обратной совместимости API). + progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента. + """ + trace_id = self._new_trace_id("sync-segmented") + started_at = time.perf_counter() + base_url = self.settings.listing.cars_url + _ = start_page # обратная совместимость — page-level resume удалён + + total_cars_upserted = 0 + total_cars_failed = 0 + total_images_upserted = 0 + total_skipped = 0 + total_discovered = 0 + all_failures: list[dict[str, str]] = [] + segment_results: list[dict[str, Any]] = [] + completed_all = True + skipped_segments = 0 + + # В segmented-режиме полный прогон должен проходить ВСЕ сегменты. + # Runtime-фильтры применяются позже (на уровне конкретных карточек), + # но не должны сужать сам обход сегментов. + + logger.info( + "Starting segmented sync: %d segments, resume from segment=%d", + len(segments), start_segment, + ) + + for seg_idx in range(start_segment, len(segments)): + seg = segments[seg_idx] + seg_make = seg.get("make") + seg_year_min = seg.get("year_min") + seg_year_max = seg.get("year_max") + seg_listing_url = seg.get("listing_url") + + # На длительном full-scan сегмент нельзя пропускать из-за runtime include.brands, + # иначе прогон становится частичным. + self._reload_runtime_config() + + if seg_listing_url: + seg_url = str(seg_listing_url) + else: + seg_url = None if self.settings.scraping_profile.http_first else (self._build_segment_listing_url(base_url, seg_make) if seg_make else None) + + seg_label = f"{seg_make or 'ALL'}" + if seg_listing_url: + seg_label = "FILTERED_SEARCH" + if seg_year_min is not None or seg_year_max is not None: + seg_label += f" ({seg_year_min}-{seg_year_max})" + + logger.debug( + "Segment %d/%d started: %s", + seg_idx + 1, len(segments), seg_label, + ) + + outer_progress_callback = self._progress_callback + + def _segment_progress(stage: str, meta: dict[str, Any]) -> None: + if outer_progress_callback is None: + return + outer_progress_callback( + stage, + { + **meta, + "segment_index": seg_idx, + "segment_label": seg_label, + "segments_total": len(segments), + }, + ) + + try: + self.set_progress_callback(_segment_progress) + self._report_progress( + "segment_started", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + ) + result = self.sync_listing( + make=seg_make, + model=None, + lane=lane, + only_new=only_new, + listing_url=seg_url, + year_min=seg_year_min, + year_max=seg_year_max, + ) + total_cars_upserted += result.get("cars_upserted", 0) + total_cars_failed += result.get("cars_failed", 0) + total_images_upserted += result.get("images_upserted", 0) + total_skipped += result.get("skipped_existing", 0) + total_discovered += result.get("listing", {}).get("vehicles_collected", 0) + all_failures.extend(result.get("failures", [])) + segment_results.append({ + "segment": seg, + "segment_index": seg_idx, + "status": result.get("status"), + "full_scan_completed": bool(result.get("full_scan_completed", False)), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), + }) + + segment_done = bool(result.get("full_scan_completed", False)) + # Даже если сегмент завершился неполно (например, страница/пагинация сломалась), + # в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше. + if not segment_done: + completed_all = False + skipped_segments += 1 + logger.info( + "Segment %d/%d incomplete: %s — advancing checkpoint and continuing", + seg_idx + 1, len(segments), seg_label, + ) + + # Сегмент обработан до конечного состояния — фиксируем чекпоинт, + # даже если он был пропущен/оборван с ошибками. + if progress_callback is not None: + try: + progress_callback(seg_idx) + except Exception: + logger.warning( + "Failed to persist segment checkpoint for segment=%d", + seg_idx, exc_info=True, + ) + + logger.info( + "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d", + seg_idx + 1, len(segments), seg_label, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + result.get("listing", {}).get("vehicles_collected", 0), + ) + self._report_progress( + "segment_done", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + segment_status=result.get("status"), + segment_full_scan_completed=segment_done, + ) + except Exception as exc: + logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc) + completed_all = False + all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)}) + skipped_segments += 1 + segment_results.append({ + "segment": seg, + "segment_index": seg_idx, + "status": "skipped_error", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "vehicles_collected": 0, + }) + if progress_callback is not None: + try: + progress_callback(seg_idx) + except Exception: + logger.warning( + "Failed to persist skipped segment checkpoint for segment=%d", + seg_idx, exc_info=True, + ) + self._report_progress( + "segment_failed", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + error=str(exc), + ) + # Продолжаем оставшиеся сегменты — одна ошибка не должна убивать весь прогон + continue + finally: + self.set_progress_callback(outer_progress_callback) + + elapsed = round(time.perf_counter() - started_at, 3) + status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed" + + logger.info( + "Segmented sync done: %d/%d segments, upserted=%d, failed=%d, discovered=%d, elapsed=%.1fs", + len(segment_results), len(segments), + total_cars_upserted, total_cars_failed, total_discovered, elapsed, + ) + + return { + "trace_id": trace_id, + "status": status, + # Bootstrap считается завершённым если все сегменты пройдены, + # даже если часть машин failed (они будут обновлены в следующих циклах). + "full_scan_completed": completed_all, + "segments_total": len(segments), + "segments_completed": len(segment_results), + "segments_skipped": skipped_segments, + "cars_upserted": total_cars_upserted, + "cars_failed": total_cars_failed, + "images_upserted": total_images_upserted, + "skipped_existing": total_skipped, + "total_discovered": total_discovered, + "elapsed_seconds": elapsed, + "failures": all_failures, + "segment_results": segment_results, + } + + def run_scheduled(self) -> None: + """Standalone-планировщик (в продакшене используется Celery beat).""" + def _handle_shutdown(signum, frame): + logger.info("Received signal %s, shutting down gracefully...", signum) + self._shutdown_requested = True + + signal.signal(signal.SIGINT, _handle_shutdown) + signal.signal(signal.SIGTERM, _handle_shutdown) + + interval = self.settings.scheduler_interval_minutes * 60 + logger.info( + "Scheduler started: syncing every %d minutes", + self.settings.scheduler_interval_minutes, + ) + cycle = 0 + while not self._shutdown_requested: + cycle += 1 + logger.info("Scheduler cycle #%d starting", cycle) + start = time.time() + try: + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = None + + if self.browser is None or self.playwright is None: + logger.info("Browser/Playwright not available, re-initializing...") + self.close() + self.__enter__() + + result = self.sync_listing() + elapsed = time.time() - start + logger.info( + "Cycle #%d done in %.1fs: %d upserted, %d failed", + cycle, elapsed, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + ) + except Exception as exc: + elapsed = time.time() - start + logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) + try: + self.close() + except Exception: + pass + try: + self.__enter__() + except Exception as reinit_exc: + logger.error("Failed to re-initialize browser: %s", reinit_exc) + + if self._shutdown_requested: + break + + sleep_time = max(0, interval - (time.time() - start)) + if sleep_time > 0: + logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) + slept = 0.0 + while slept < sleep_time and not self._shutdown_requested: + chunk = min(5.0, sleep_time - slept) + time.sleep(chunk) + slept += chunk + + logger.info("Scheduler stopped gracefully after %d cycles.", cycle) diff --git a/iaai_scraper/storage/__init__.py b/iaai_scraper/storage/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/storage/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py new file mode 100644 index 0000000..ffb69ca --- /dev/null +++ b/iaai_scraper/storage/db.py @@ -0,0 +1,676 @@ +import logging +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Any, Iterator + +from sqlalchemy import create_engine, delete, or_, select, text, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session, sessionmaker + +from ..core.config import Settings +from .models import Base, Car, Image, SyncRun +from .schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.db") + + +CAR_DB_FIELDS = { + col.key for col in Car.__table__.columns + if col.key not in ("id",) +} + +_IN_CHUNK_SIZE = 5000 +CAR_TABLE_NAME = Car.__tablename__ + + +class PersistenceService: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + engine_kwargs = { + "echo": settings.database.echo, + "future": True, + } + if "postgresql" in settings.database.url: + engine_kwargs["pool_size"] = settings.database.pool_size + engine_kwargs["max_overflow"] = settings.database.max_overflow + engine_kwargs["pool_pre_ping"] = True + engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds + engine_kwargs["pool_timeout"] = 30 + # Таймауты запросов и блокировок. + engine_kwargs["connect_args"] = { + "options": "-c statement_timeout=120000 -c lock_timeout=30000" + } + self.engine = create_engine(settings.database.url, **engine_kwargs) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) + + def create_tables(self) -> None: + # Для SQLite можно create_all. + is_sqlite = self.settings.database.url.startswith("sqlite") + if not is_sqlite and not self.settings.database.auto_create_tables: + return + try: + Base.metadata.create_all(self.engine) + except Exception: + logger.debug("create_tables skipped (schema already exists)") + + @contextmanager + def session_scope(self) -> Iterator[Session]: + session = self.session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def start_sync_run(self, lane: str) -> int: + with self.session_scope() as session: + now = datetime.now(timezone.utc) + stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all() + for stale in stale_runs: + stale.status = "failed" + stale.finished_at = now + if not stale.error_summary: + stale.error_summary = "Recovered stale running sync run before starting a new run" + + run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) + session.add(run) + session.flush() + return int(run.id) + + def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + with self.session_scope() as session: + run = session.get(SyncRun, run_id) + if run is None: + return + run.finished_at = datetime.now(timezone.utc) + run.status = status + run.ids_fetched = ids_fetched + run.cars_upserted = cars_upserted + run.cars_failed = cars_failed + run.images_upserted = images_upserted + run.error_summary = error_summary + + def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]: + if not origin_urls: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: + if not origin_ids: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_urls_and_ids( + self, origin_urls: list[str], origin_ids: list[str], + ) -> tuple[set[str], set[str]]: + # Загрузка URL и origin_id. + if not origin_urls and not origin_ids: + return set(), set() + urls: set[str] = set() + ids: set[str] = set() + with self.session_scope() as session: + max_len = max(len(origin_urls), len(origin_ids), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + conditions = [] + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car.origin_url, Car.origin_id).where(or_(*conditions)) + ).all() + for r in rows: + if r[0]: + urls.add(str(r[0])) + if r[1]: + ids.add(str(r[1])) + return urls, ids + + @staticmethod + def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: + if not images: + return + session.add_all([ + Image( + fullres_image=str(img["fullres_image"]), + preview_image=str(img["preview_image"]), + order_index=int(img.get("order_index", 0)), + car_id=car_id, + ) + for img in images + ]) + + @staticmethod + def _car_payload(record: CarRecord) -> dict[str, object]: + payload = record.model_dump(mode="python") + return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + + def _is_postgres(self) -> bool: + return self.engine.dialect.name == "postgresql" + + def _load_existing_cars( + self, + session: Session, + origin_ids: list[str], + origin_urls: list[str], + ) -> tuple[dict[str, Car], dict[str, Car]]: + existing_by_id: dict[str, Car] = {} + existing_by_url: dict[str, Car] = {} + if not origin_ids and not origin_urls: + return existing_by_id, existing_by_url + + existing_cars: list[Car] = [] + max_len = max(len(origin_ids), len(origin_urls), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + conditions = [] + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car).where(or_(*conditions)) + ).scalars().all() + existing_cars.extend(rows) + + for car in existing_cars: + if car.origin_id: + existing_by_id[car.origin_id] = car + if car.origin_url: + existing_by_url[car.origin_url] = car + return existing_by_id, existing_by_url + + def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]: + existing_images_map: dict[int, set[str]] = {} + if not car_ids: + return existing_images_map + + car_id_list = list(car_ids) + for i in range(0, len(car_id_list), _IN_CHUNK_SIZE): + chunk = car_id_list[i:i + _IN_CHUNK_SIZE] + img_rows = session.execute( + select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk)) + ).all() + for cid, furl in img_rows: + existing_images_map.setdefault(int(cid), set()).add(str(furl)) + return existing_images_map + + @staticmethod + def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]: + return { + key: getattr(insert_stmt.excluded, key) + for key in CAR_DB_FIELDS + } + + def _replace_images_for_car( + self, + session: Session, + car_id: int, + images: list[dict[str, object]], + origin_id: str, + ) -> int: + nested = session.begin_nested() + try: + session.execute(delete(Image).where(Image.car_id == car_id)) + self._add_images(session, car_id, images) + session.flush() + nested.commit() + return len(images) + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True) + return 0 + + def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]: + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + entries: list[dict[str, object]] = [] + upsert_payloads: list[dict[str, object]] = [] + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + car_by_id = existing_by_id.get(record.origin_id) + car_by_url = existing_by_url.get(record.origin_url) + entry: dict[str, object] = {"record": record, "images": images, "car_id": None} + + if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + entry["car_id"] = int(car_by_url.id) + updated += 1 + else: + upsert_payloads.append(payload) + if car_by_id is not None: + updated += 1 + else: + inserted += 1 + entries.append(entry) + + session.flush() + + if upsert_payloads: + insert_stmt = pg_insert(Car).values(upsert_payloads) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id, Car.origin_id) + rows = session.execute(upsert_stmt).all() + car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows} + for entry in entries: + if entry["car_id"] is not None: + continue + record = entry["record"] + car_id = car_ids_by_origin_id.get(record.origin_id) + if car_id is None: + raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}") + entry["car_id"] = car_id + + car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None} + existing_images_map = self._load_existing_image_urls(session, car_ids) + images_by_car_id: dict[int, list[dict[str, object]]] = {} + replace_ids: list[int] = [] + + for entry in entries: + car_id = int(entry["car_id"]) + images = entry["images"] + new_image_urls = { + str(img.get("fullres_image", "")) + for img in images + if img.get("fullres_image") + } + old_image_urls = existing_images_map.get(car_id, set()) + if new_image_urls != old_image_urls: + replace_ids.append(car_id) + images_by_car_id[car_id] = images + else: + images_total += len(old_image_urls) + + if replace_ids: + for i in range(0, len(replace_ids), _IN_CHUNK_SIZE): + chunk = replace_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car_id in replace_ids: + images = images_by_car_id[car_id] + self._add_images(session, car_id, images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def upsert_car(self, record: CarRecord): + # Вставка или обновление авто. + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + with self.session_scope() as session: + if self._is_postgres(): + car_by_url = session.execute( + select(Car).where(Car.origin_url == record.origin_url) + ).scalar_one_or_none() + if car_by_url is not None and car_by_url.origin_id != record.origin_id: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + session.flush() + car_id = int(car_by_url.id) + action = "updated" + else: + existed = session.execute( + select(Car.id).where(Car.origin_id == record.origin_id) + ).scalar_one_or_none() is not None + insert_stmt = pg_insert(Car).values(**payload) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id) + car_id = int(session.execute(upsert_stmt).scalar_one()) + action = "updated" if existed or car_by_url is not None else "inserted" + + images_upserted = self._replace_images_for_car( + session, car_id, images, record.origin_id, + ) + return {"car_id": car_id, "images_upserted": images_upserted, "action": action} + + car = session.execute( + select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) + ).scalar_one_or_none() + action = "inserted" + if car is None: + car = Car(**payload) + session.add(car) + session.flush() + else: + action = "updated" + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + session.flush() + images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id) + return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action} + self._add_images(session, int(car.id), images) + session.flush() + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]: + """Пакетный upsert нескольких автомобилей в одной транзакции. + + Оптимизации: + - Дедупликация записей по origin_id перед вставкой. + - Chunked IN-queries для больших списков (обход лимита PG параметров). + - Пропуск перезаписи изображений, если набор URL не изменился. + - Один DELETE по car_id IN (...) вместо удаления по одному. + - Fallback на по-одному upsert если batch commit упал. + """ + # Дедупликация батча. + seen_ids: dict[str, int] = {} + unique_records: list[CarRecord] = [] + for idx, r in enumerate(records): + key = r.origin_id or r.origin_url + if key in seen_ids: + logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key]) + continue + seen_ids[key] = idx + unique_records.append(r) + + if len(unique_records) < len(records): + logger.info("Deduped batch: %d → %d records", len(records), len(unique_records)) + records = unique_records + + try: + return self._upsert_cars_batch_inner(records) + except Exception as exc: + logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc) + return self._upsert_cars_individually(records) + + def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]: + """Внутренняя реализация batched upsert (одна транзакция).""" + if self._is_postgres(): + return self._upsert_cars_batch_postgres(records) + + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + # Загружаем существующие записи. + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + # Предзагружаем изображения. + existing_car_ids = set() + for record in records: + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is not None: + existing_car_ids.add(int(car.id)) + + # Готовим map car_id -> image_urls. + existing_images_map = self._load_existing_image_urls(session, existing_car_ids) + + new_cars: list[tuple[Car, list[dict]]] = [] + update_cars_needing_images: list[tuple[Car, list[dict]]] = [] + + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is None: + car = Car(**payload) + session.add(car) + inserted += 1 + new_cars.append((car, images)) + else: + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + updated += 1 + + # Проверяем изменения картинок. + new_image_urls = {img.get("fullres_image", "") for img in images} + old_image_urls = existing_images_map.get(int(car.id), set()) + if new_image_urls != old_image_urls: + update_cars_needing_images.append((car, images)) + else: + images_total += len(old_image_urls) + + # Один flush. + session.flush() + + # Добавляем картинки новым авто. + for car, images in new_cars: + self._add_images(session, int(car.id), images) + images_total += len(images) + + # Обновляем только изменённые картинки. + if update_cars_needing_images: + update_ids = [int(car.id) for car, _ in update_cars_needing_images] + for i in range(0, len(update_ids), _IN_CHUNK_SIZE): + chunk = update_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car, images in update_cars_needing_images: + self._add_images(session, int(car.id), images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]: + # Запасной поштучный upsert. + inserted = 0 + updated = 0 + images_total = 0 + for record in records: + try: + result = self.upsert_car(record) + action = result.get("action", "inserted") + if action == "inserted": + inserted += 1 + else: + updated += 1 + images_total += int(result.get("images_upserted", 0)) + except Exception as exc: + logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int: + """Помечает авто как проданные, если их нет в активном листинге (по origin_id).""" + if not active_origin_ids: + return 0 + with self.session_scope() as session: + stmt = ( + update(Car) + .where(Car.origin_id.notin_(active_origin_ids)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("iaai:%")) + .values(is_sold=True) + ) + result = session.execute(stmt) + count = result.rowcount or 0 + if count: + logger.info("Marked %d cars as sold (no longer in listing)", count) + return count + + def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int: + """Помечает авто как проданные, если их URL нет в активном листинге. + + Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, + что кардинально быстрее при больших объёмах (100K+ URLs). + Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент. + """ + if not active_origin_urls: + return 0 + + # Нормализация URL. + def _norm(url: str) -> str: + return url.replace("~", "-") + + normalized_urls = {_norm(u) for u in active_origin_urls} + + is_postgres = "postgresql" in self.settings.database.url + + with self.session_scope() as session: + if is_postgres: + # Считаем кандидатов на sold. + total_active = session.execute( + text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'") + ).scalar() or 0 + + if total_active == 0: + return 0 + + # Временная таблица URL. + session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP")) + session.execute(text("TRUNCATE _active_urls")) + + # Вставляем URL чанками. + url_list = list(normalized_urls) + for i in range(0, len(url_list), _IN_CHUNK_SIZE): + chunk = url_list[i:i + _IN_CHUNK_SIZE] + values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) + params = {f"u{j}": url for j, url in enumerate(chunk)} + session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params) + + # Индекс для JOIN. + session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)")) + + # Считаем будущие sold. + would_mark = session.execute(text(""" + SELECT count(*) + FROM {car_table} c + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'iaai:%%' + """.format(car_table=CAR_TABLE_NAME))).scalar() or 0 + + # Защита от аномалии. + if total_active > 100 and would_mark > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + would_mark, total_active, would_mark / total_active * 100, + ) + return 0 + + # Массовая пометка sold. + result = session.execute(text(""" + UPDATE {car_table} + SET is_sold = TRUE + FROM ( + SELECT c.id + FROM {car_table} c + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'iaai:%%' + ) sub + WHERE {car_table}.id = sub.id + """.format(car_table=CAR_TABLE_NAME))) + count = result.rowcount or 0 + else: + # Упрощённый путь для SQLite. + stmt = ( + update(Car) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("iaai:%")) + .values(is_sold=True) + ) + # Загружаем active URL. + all_active = session.execute( + select(Car.id, Car.origin_url).where( + Car.is_sold == False, Car.origin_id.like("iaai:%") # noqa: E712 + ) + ).all() + mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls] + + if not mark_ids: + return 0 + total_active = len(all_active) + if total_active > 100 and len(mark_ids) > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + len(mark_ids), total_active, len(mark_ids) / total_active * 100, + ) + return 0 + + for i in range(0, len(mark_ids), _IN_CHUNK_SIZE): + chunk = mark_ids[i:i + _IN_CHUNK_SIZE] + session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True)) + count = len(mark_ids) + + if count: + logger.info("Marked %d cars as sold by URL (no longer in listing)", count) + return count + + def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]: + """Возвращает все известные origin_id для заданного префикса. + + Использует yield_per для потоковой загрузки при большом количестве записей. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} + + def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]: + """Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса.""" + with self.session_scope() as session: + result = session.execute( + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} + + def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int: + """Возвращает количество активных (не проданных) авто для заданного lane-префикса.""" + from sqlalchemy import func as sa_func + with self.session_scope() as session: + result = session.execute( + select(sa_func.count()).select_from(Car).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ) + ) + return int(result.scalar() or 0) + + def get_active_origin_urls_batch_for_refresh( + self, + prefix: str = "iaai:", + offset: int = 0, + limit: int = 500, + ) -> list[str]: + """Возвращает батч origin_url активных авто для rolling refresh. + + Сортировка по last_seen_at ASC — давно не обновлённые идут первыми. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit) + ) + return [str(row[0]) for row in result if row and row[0]] diff --git a/iaai_scraper/storage/enums.py b/iaai_scraper/storage/enums.py new file mode 100644 index 0000000..ed74afb --- /dev/null +++ b/iaai_scraper/storage/enums.py @@ -0,0 +1,25 @@ +CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") +DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA") +GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") +STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA") +BODY_TYPE_ENUM_VALUES = ( + "COUPE", + "SUV", + "HATCHBACK", + "MINIVAN", + "SEDAN", + "STATION_WAGON", + "PICKUP", + "TRUCK", + "OPEN", + "RV", + "OTHER", + "NA", +) +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") +ORIGIN_ENUM_VALUES = ( + "IAAI", + "MOBILE_DE", + "NA", +) +SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "CLASSIFIED", "NA") diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py new file mode 100644 index 0000000..1970dd0 --- /dev/null +++ b/iaai_scraper/storage/models.py @@ -0,0 +1,82 @@ +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from .enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) + + +class Base(DeclarativeBase): + pass + + +class Car(Base): + __tablename__ = "iaai_cars" + __table_args__ = ( + Index("ix_iaai_cars_brand_model", "brand", "model"), + Index("ix_iaai_cars_origin_id_not_sold", "origin_id", "is_sold"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + model: Mapped[str] = mapped_column(String(50), nullable=False) + year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) + price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD") + mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + color: Mapped[str] = mapped_column(String(), nullable=False, default="other") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER") + engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) + is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) + non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + slug: Mapped[str] = mapped_column(String(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) + images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") + + +class Image(Base): + __tablename__ = "iaai_images" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + fullres_image: Mapped[str] = mapped_column(String(), nullable=False) + preview_image: Mapped[str] = mapped_column(String(), nullable=False) + order_index: Mapped[int] = mapped_column(Integer, nullable=False) + car_id: Mapped[int] = mapped_column(Integer, ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False, index=True) + car: Mapped[Car] = relationship("Car", back_populates="images") + + +class SyncRun(Base): + __tablename__ = "iaai_sync_runs" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False, index=True) + lane: Mapped[str] = mapped_column(Text, nullable=False) + ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/iaai_scraper/storage/schemas.py b/iaai_scraper/storage/schemas.py new file mode 100644 index 0000000..7af7c43 --- /dev/null +++ b/iaai_scraper/storage/schemas.py @@ -0,0 +1,87 @@ +from datetime import datetime, timezone +from pydantic import BaseModel, ConfigDict, Field + + +class ImageRecord(BaseModel): + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRecord(BaseModel): + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str + origin_id: str + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str + last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + images: list[ImageRecord] = Field(default_factory=list) + + +class ImageRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str = "" + origin_id: str = "" + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str = "" + last_seen_at: datetime | None = None + images: list[ImageRead] = Field(default_factory=list) + diff --git a/iaai_scraper/worker/__init__.py b/iaai_scraper/worker/__init__.py new file mode 100644 index 0000000..841be13 --- /dev/null +++ b/iaai_scraper/worker/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/iaai_scraper/worker/celery_app.py b/iaai_scraper/worker/celery_app.py new file mode 100644 index 0000000..bb76020 --- /dev/null +++ b/iaai_scraper/worker/celery_app.py @@ -0,0 +1,207 @@ +# Инициализация Celery-приложения и периодических задач. + +import json +import logging +import os +import time + +from celery import Celery +from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging +from redis import Redis + +from ..core.config import settings +from ..core.logs import setup_logging + +logger = logging.getLogger("iaai_scraper.worker.celery_app") +STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched" +IAAI_SYNC_QUEUE = "iaai_sync" +MOBILEDE_SYNC_QUEUE = "mobilede_sync" +PROGRESS_KEY_PREFIX = "iaai:state:task_progress:" + + +def _env_bool(name: str, default: bool) -> bool: + raw = os.getenv(name, "true" if default else "false").strip().lower() + return raw in {"1", "true", "yes", "on"} + + +def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool: + now = int(time.time()) + try: + for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"): + payload = redis_client.get(raw_key) + if not payload: + continue + try: + progress = json.loads(payload) + except (TypeError, ValueError): + continue + ts = int(progress.get("ts") or 0) + stage = str(progress.get("stage") or "") + if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}: + return True + except Exception: + logger.warning("Failed to inspect startup progress keys", exc_info=True) + return False + + +@celery_setup_logging.connect +def _configure_logging(loglevel=None, **kwargs): + # Перехватываем логирование Celery и пишем только в stderr (Docker logs). + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +@worker_process_init.connect +def _on_worker_process_init(**kwargs): + # Повторно настраиваем логирование в каждом дочернем prefork-процессе, + # чтобы StreamHandler(stderr) корректно работал после fork. + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +def _broker_url() -> str: + return settings.celery.broker_url or settings.redis.url + + +def _result_backend() -> str: + return settings.celery.result_backend or settings.redis.url + + +celery_app = Celery( + "iaai_scraper", + broker=_broker_url(), + backend=_result_backend(), +) + +# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120), +# ограничиваем, чтобы зависший worker не жил вечно. +_soft = settings.celery.task_soft_time_limit +_hard = settings.celery.task_time_limit +_max_hard = _soft + 120 if _soft else _hard +if _hard > _max_hard: + logger.info( + "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; " + "clamping hard limit to %d", + _hard, _soft, _max_hard, + ) + _hard = _max_hard + +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_soft_time_limit=_soft, + task_time_limit=_hard, + task_acks_late=True, + task_reject_on_worker_lost=True, + task_track_started=True, + worker_concurrency=settings.celery.worker_concurrency, + worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child, + worker_pool=settings.celery.worker_pool, + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + broker_transport_options={ + "visibility_timeout": settings.celery.broker_visibility_timeout, + }, + result_expires=86400, + worker_redirect_stdouts=False, + worker_hijack_root_logger=False, + beat_schedule={ + "periodic-mobilede-sync-search": { + "task": "mobilede.sync_runtime_segments", + "schedule": settings.celery.beat_sync_interval_minutes * 60.0, + "args": (), + "kwargs": { + "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")), + "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True), + "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True), + }, + "options": { + "queue": MOBILEDE_SYNC_QUEUE, + "expires": settings.celery.beat_sync_interval_minutes * 60.0, + }, + } + }, + task_routes={ + "mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE}, + "mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE}, + "mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE}, + "iaai.sync_cars_feed": {"queue": IAAI_SYNC_QUEUE}, + "iaai_scraper.worker.tasks.*": {"queue": IAAI_SYNC_QUEUE}, + }, +) + +celery_app.autodiscover_tasks(["iaai_scraper.worker"]) + + +@worker_ready.connect +def _on_worker_ready(**kwargs): + """При старте worker отправляем первый sync_listing, если очередь пуста.""" + if not _env_bool("IAAI_STARTUP_SYNC_ENABLED", True): + logger.info("Worker ready: startup sync dispatch disabled by IAAI_STARTUP_SYNC_ENABLED") + return + + redis_client = None + try: + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + has_fresh_progress = _has_fresh_active_progress(redis_client) + for stale_key in ("iaai:locks:sync_listing",): + try: + ttl = redis_client.ttl(stale_key) + if ttl is not None and ttl != -2 and not has_fresh_progress: + redis_client.delete(stale_key) + logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl) + elif ttl is not None and ttl != -2: + logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl) + except Exception: + logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True) + + try: + queue_len = int(redis_client.llen(IAAI_SYNC_QUEUE) or 0) + except Exception: + queue_len = 0 + if queue_len > 0: + logger.info("Worker ready: iaai_sync queue already has %d task(s); skip startup dispatch", queue_len) + return + + should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) + if not should_dispatch and not has_fresh_progress: + redis_client.delete(STARTUP_SYNC_DISPATCH_KEY) + should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) + if should_dispatch: + logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists") + except Exception: + logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True) + return + finally: + if redis_client is not None: + try: + redis_client.close() + except Exception: + pass + + if not should_dispatch: + logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue") + return + + logger.info("Worker ready — dispatching initial mobile.de sync_search task") + celery_app.send_task( + "mobilede.sync_runtime_segments", + kwargs={ + "delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")), + "use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True), + "continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True), + }, + queue=MOBILEDE_SYNC_QUEUE, + expires=settings.celery.beat_sync_interval_minutes * 60.0, + ) diff --git a/iaai_scraper/worker/self_heal.py b/iaai_scraper/worker/self_heal.py new file mode 100644 index 0000000..8641706 --- /dev/null +++ b/iaai_scraper/worker/self_heal.py @@ -0,0 +1,276 @@ +import json +import logging +import os +import random +import signal +import time + +from redis import Redis + + +logger = logging.getLogger("iaai_scraper.worker.self_heal") + +IAAI_SYNC_QUEUE = "iaai_sync" +GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts" +GLOBAL_DB_PROGRESS_TS_KEY = "iaai:state:last_db_progress_ts" +SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress" +SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing" +SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done" +SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint" +SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending" +SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM) + + +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_int(name: str, default: int) -> int: + value = os.getenv(name) + if value is None: + return default + try: + return int(value.strip()) + except Exception: + return default + + +def _get_redis() -> Redis: + url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0") + return Redis.from_url( + url, + decode_responses=True, + socket_connect_timeout=5.0, + socket_timeout=10.0, + health_check_interval=30, + retry_on_timeout=True, + ) + + +def _safe_int(value: str | None, default: int = 0) -> int: + if value is None: + return default + try: + return int(str(value).strip()) + except Exception: + return default + + +def _read_last_progress_ts(redis_client: Redis) -> int | None: + raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY) + if raw: + ts = _safe_int(raw) + if ts > 0: + return ts + + # Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*. + # Это дороже, но выполняется только при отсутствии основного маркера. + max_ts = 0 + for key in redis_client.scan_iter(match="iaai:state:task_progress:*"): + try: + payload = redis_client.get(key) + if not payload: + continue + data = json.loads(payload) + ts = _safe_int(data.get("ts"), 0) + if ts > max_ts: + max_ts = ts + except Exception: + continue + return max_ts or None + + +def _read_last_db_progress_ts(redis_client: Redis) -> int | None: + raw = redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) + if raw: + ts = _safe_int(raw) + if ts > 0: + return ts + + max_ts = 0 + for key in redis_client.scan_iter(match="iaai:state:task_progress:*"): + try: + payload = redis_client.get(key) + if not payload: + continue + data = json.loads(payload) + if str(data.get("stage") or "") == "fast_db_progress": + ts = _safe_int(data.get("ts"), 0) + else: + ts = _safe_int(data.get("last_db_progress_ts"), 0) + if ts > max_ts: + max_ts = ts + except Exception: + continue + return max_ts or None + + +def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None: + pipe = redis_client.pipeline() + pipe.delete(SYNC_LISTING_CHECKPOINT_KEY) + pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) + pipe.delete(GLOBAL_PROGRESS_TS_KEY) + pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY) + pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0") + pipe.execute() + + +def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]: + """Есть ли признаки активной/зависшей работы, даже если очередь пуста.""" + queue_len = _safe_int(redis_client.llen(queue_name), 0) + has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0 + has_task_progress = 0 + for _ in redis_client.scan_iter(match="iaai:state:task_progress:*"): + has_task_progress = 1 + break + flags = { + "queue_len": queue_len, + "has_lock": has_lock, + "has_task_progress": has_task_progress, + } + return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags + + +def _kill_worker_process() -> None: + pid_file = "/tmp/celery-worker.pid" + pid: int | None = None + try: + with open(pid_file, "r", encoding="utf-8") as f: + pid = int(f.read().strip()) + except Exception: + pid = None + + if not pid: + logger.error("Self-heal: failed to read worker pid from %s", pid_file) + return + + logger.error("Self-heal: terminating stuck worker process pid=%s", pid) + try: + os.kill(pid, signal.SIGTERM) + except Exception: + logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid) + return + + time.sleep(20) + try: + # Если процесс ещё жив — принудительно убиваем. + os.kill(pid, 0) + logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid) + os.kill(pid, SIGKILL_FALLBACK) + except ProcessLookupError: + pass + except Exception: + logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid) + + +def main() -> None: + if not _env_bool("IAAI_SELF_HEAL_ENABLED", True): + logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED") + return + + queue_name = os.getenv("IAAI_CELERY_QUEUE", IAAI_SYNC_QUEUE) + check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30)) + stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720)) + db_idle_seconds = max(60, _env_int("IAAI_DB_IDLE_RESTART_SECONDS", 3600)) + startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300)) + restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300)) + + logger.info( + "Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss db_idle=%ss startup_grace=%ss cooldown=%ss", + queue_name, + check_interval, + stall_seconds, + db_idle_seconds, + startup_grace, + restart_cooldown, + ) + + started_at = time.time() + redis_client: Redis | None = None + + while True: + try: + if redis_client is None: + redis_client = _get_redis() + redis_client.ping() + + has_inflight, inflight = _has_inflight_work(redis_client, queue_name) + if not has_inflight: + time.sleep(check_interval) + continue + + last_progress_ts = _read_last_progress_ts(redis_client) + now_ts = int(time.time()) + age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts)) + + if age is None: + if now_ts - int(started_at) < startup_grace: + time.sleep(check_interval) + continue + logger.warning( + "Self-heal: inflight=%s but no progress timestamp found after startup grace", + inflight, + ) + age = stall_seconds + 1 + + restart_reason = f"progress_age={age}s > {stall_seconds}s" + db_idle_restart = False + if age <= stall_seconds: + last_db_ts = _read_last_db_progress_ts(redis_client) + db_age = None if last_db_ts is None else max(0, now_ts - int(last_db_ts)) + if db_age is None: + first_allowed_ts = int(started_at) + max(startup_grace, db_idle_seconds) + if now_ts < first_allowed_ts: + time.sleep(check_interval) + continue + db_age = db_idle_seconds + 1 + if db_age <= db_idle_seconds: + time.sleep(check_interval) + continue + db_idle_restart = True + restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s" + + # Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно. + acquired = bool( + redis_client.set( + SELF_HEAL_RESTART_LOCK_KEY, + str(now_ts), + nx=True, + ex=restart_cooldown, + ) + ) + if not acquired: + time.sleep(check_interval) + continue + + logger.error( + "Self-heal: detected stall (inflight=%s, %s). Restarting worker process...", + inflight, + restart_reason, + ) + if db_idle_restart: + logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1") + _reset_bootstrap_checkpoint_for_db_idle(redis_client) + # Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров + # перезапуск был не строго одновременно. + time.sleep(random.uniform(0.3, 2.0)) + _kill_worker_process() + # После kill pid1 контейнер будет перезапущен Docker restart-policy. + # На случай неуспеха не молотим цикл. + time.sleep(check_interval) + + except Exception: + logger.exception("Self-heal watchdog iteration failed") + redis_client = None + time.sleep(check_interval) + + +if __name__ == "__main__": + logging.basicConfig( + level=os.getenv("IAAI_LOG_LEVEL", "INFO"), + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + ) + main() diff --git a/iaai_scraper/worker/tasks.py b/iaai_scraper/worker/tasks.py new file mode 100644 index 0000000..06522ab --- /dev/null +++ b/iaai_scraper/worker/tasks.py @@ -0,0 +1,3358 @@ +# Задачи Celery для синхронизации автомобилей и листинга IAAI. + +import json +import logging +import os +import signal +import hashlib +from threading import Event, Thread +import time +import uuid +from urllib.parse import parse_qsl, urlsplit + +from billiard.exceptions import SoftTimeLimitExceeded +from celery import shared_task +from redis import Redis +import requests + +from ..core.config import Settings, build_fast_listing_segments_for_makes, build_listing_segments_for_makes, parse_listing_segments +from ..core.runtime_config import RuntimeConfig +from ..mobile_de import MobileDeClient, MobileDeScraper +from ..scraper import IAAIScraper +from ..storage.db import PersistenceService +from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats + +logger = logging.getLogger("iaai_scraper.worker.tasks") + +IAAI_SYNC_QUEUE = "iaai_sync" +MOBILEDE_SYNC_QUEUE = "mobilede_sync" +MOBILEDE_SEARCH_CURSOR_KEY = "mobilede:state:search_next_page" +MOBILEDE_SEGMENT_CURSOR_KEY_FMT = "mobilede:state:search_next_page:{segment_key}" +MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY = "mobilede:state:runtime_segment_index" +MOBILEDE_RUNTIME_SEGMENTS_TASK = "mobilede.sync_runtime_segments" +MOBILEDE_PROGRESS_PAGE_COUNTER_KEY_FMT = "mobilede:state:progress_pages:{segment_key}" +MOBILEDE_CONTINUOUS_SYNC_ENABLED = os.getenv("MOBILEDE_CONTINUOUS_SYNC_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS", "15")))) +MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "50"))) +MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_RUNTIME_INITIAL_TASKS = max(1, int(os.getenv("MOBILEDE_RUNTIME_INITIAL_TASKS", "2"))) +MOBILEDE_SEGMENT_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_SEGMENT_PAGE_WINDOW", "10"))) +MOBILEDE_SEGMENT_TARGET_RESULTS = max(100, int(os.getenv("MOBILEDE_SEGMENT_TARGET_RESULTS", "1800"))) +MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "false").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED = os.getenv("MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP = os.getenv("MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_INCREMENTAL_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_INCREMENTAL_PAGE_WINDOW", "1"))) +MOBILEDE_ONLY_NEW_NEWEST_FIRST = os.getenv("MOBILEDE_ONLY_NEW_NEWEST_FIRST", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS = os.getenv("MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK = max(1, int(os.getenv("MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK", "1"))) +MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS = max(60, int(os.getenv("MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS", "3600"))) +MOBILEDE_ONLY_NEW_HOT_ONLY = os.getenv("MOBILEDE_ONLY_NEW_HOT_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"} +MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS", "10800"))) +MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO = min(1.0, max(0.0, float(os.getenv("MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO", "0.95")))) +MOBILEDE_BOOTSTRAP_DONE_KEY = "mobilede:state:bootstrap_full_scan_done" +MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY = "mobilede:state:bootstrap_segments_total" +MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY = "mobilede:state:bootstrap_segments_done" +MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY = "mobilede:state:runtime_segments_cache" +MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY = "mobilede:state:runtime_segments_building" +MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY = "mobilede:state:runtime_segments_pending" +MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle" +MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count" +MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT = "mobilede:state:incremental_cycle_seen:{cycle_id}" + +# Минимальная пауза между батчами (секунды) — не давит IAAI. +_INTER_BATCH_DELAY = max(float(os.getenv("IAAI_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0) +# Если доля failed в батче превышает порог — прерываем (IAAI блокирует). +_FAIL_RATE_THRESHOLD = min(max(float(os.getenv("IAAI_FAIL_RATE_THRESHOLD", "0.9")), 0.0), 1.0) + +SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing" +SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done" +SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint" +SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60 +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak" +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3 +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60 +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "iaai:state:sync_listing_bootstrap_continuation_streak" +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT = max( + 1, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT", "6")), +) +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS = max( + 60, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS", str(6 * 60 * 60))), +) +HOURLY_FAILURE_STREAK_KEY = "iaai:state:hourly_failure_streak" +HOURLY_FAILURE_STREAK_LIMIT = 3 +HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # сброс через 6 часов +SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending" +SYNC_LISTING_TASK_NAME = "iaai.sync_cars_feed" +SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}" +SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress" +SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total" +SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60 +TASK_PROGRESS_KEY_FMT = "iaai:state:task_progress:{task_id}" +GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts" +GLOBAL_DB_PROGRESS_TS_KEY = "iaai:state:last_db_progress_ts" +SITEMAP_HOURLY_LAST_COUNT_KEY = "iaai:state:sitemap_hourly_last_count" +SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "iaai:state:sitemap_hourly_refresh_offset" +STALL_WATCHDOG_NAVIGATION_STAGES = { + "listing_next_page_started", + "listing_resume_progress", +} +STALL_WATCHDOG_LONG_RUNNING_STAGES = { + "fast_listing_collected", + "fast_detail_progress", +} +STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max( + 300, + int(os.getenv("STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS", "900")), +) +STALL_WATCHDOG_DETAIL_GRACE_SECONDS = max( + 600, + int(os.getenv("STALL_WATCHDOG_DETAIL_GRACE_SECONDS", "1200")), +) +DB_IDLE_RESTART_SECONDS = max(60, int(os.getenv("IAAI_DB_IDLE_RESTART_SECONDS", "3600"))) +TERMINAL_PROGRESS_STAGES = { + "segment_done", + "segment_failed", + "segment_task_completed", + "segment_task_failed", + "segment_task_soft_timeout", + "sync_done", + "failed", +} + + +def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): + last_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return func() + except Exception as exc: + last_exc = exc + if attempt >= attempts: + break + delay = base_delay_s * (2 ** (attempt - 1)) + logger.warning( + "Operation failed (attempt %d/%d): %s. Retrying in %.1fs", + attempt, + attempts, + exc, + delay, + ) + time.sleep(delay) + if last_exc is not None: + raise last_exc + + +def _run_browser_job(func, *args, **kwargs): + # С pool=solo Celery worker работает в одном процессе/потоке. + # Playwright sync API использует greenlets, которые привязаны к потоку. + # Запуск в отдельном потоке вызывает greenlet.error: cannot switch to a different thread. + # Поэтому запускаем напрямую в текущем потоке. + return func(*args, **kwargs) + + +def _sync_listing_lock_ttl_seconds() -> int: + settings = Settings() + soft = settings.celery.task_soft_time_limit + hard = settings.celery.task_time_limit + # Используем clamped hard limit (soft + 120), а не сырой task_time_limit, + # чтобы lock не висел 11 дней при CELERY_TASK_TIME_LIMIT=999999. + effective_hard = min(hard, soft + 120) if soft else hard + return max(effective_hard + 120, 300) + + +def _sync_segment_lock_ttl_seconds() -> int: + return 300 + + +def _task_progress_key(task_id: str) -> str: + return TASK_PROGRESS_KEY_FMT.format(task_id=task_id) + + +def _update_task_progress( + redis_client: Redis, + *, + task_id: str, + stage: str, + ttl_seconds: int, + **payload, +) -> None: + try: + now_ts = int(time.time()) + existing_task_started_ts: int | None = None + try: + existing_raw = redis_client.get(_task_progress_key(task_id)) + if existing_raw: + existing_payload = json.loads(existing_raw) + existing_task_started_ts = _safe_int(existing_payload.get("task_started_ts")) + except Exception: + existing_task_started_ts = None + payload.setdefault("task_started_ts", existing_task_started_ts or now_ts) + if stage != "fast_db_progress" and "last_db_progress_ts" not in payload: + last_db_progress_ts = _safe_int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY)) + if last_db_progress_ts is not None: + payload["last_db_progress_ts"] = last_db_progress_ts + data = { + "task_id": task_id, + "stage": stage, + "ts": now_ts, + **payload, + } + ttl = max(60, int(ttl_seconds)) + pipe = redis_client.pipeline() + pipe.set( + _task_progress_key(task_id), + json.dumps(data, ensure_ascii=False), + ex=ttl, + ) + # Глобальный маркер активности для внешнего guard-процесса. + # Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера + # (когда PID жив, но прогресс по задачам не двигается). + pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) + if stage == "fast_db_progress": + pipe.set(GLOBAL_DB_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) + pipe.execute() + except Exception: + logger.warning("Failed to update task progress for %s", task_id, exc_info=True) + + +def _clear_task_progress(redis_client: Redis, task_id: str) -> None: + try: + redis_client.delete(_task_progress_key(task_id)) + except Exception: + logger.warning("Failed to clear task progress for %s", task_id, exc_info=True) + + +def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int: + if stage in STALL_WATCHDOG_NAVIGATION_STAGES: + return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS) + if stage in STALL_WATCHDOG_LONG_RUNNING_STAGES: + return max(int(default_timeout), STALL_WATCHDOG_DETAIL_GRACE_SECONDS) + return int(default_timeout) + + +def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:") + + new_urls = [url for url in discovered_urls if url not in existing_urls] + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai") + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if new_urls: + with IAAIScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + batch_size = settings.celery.batch_size + for batch_start in range(0, len(new_urls), batch_size): + batch_urls = new_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + # Fail-fast: если слишком много ошибок — IAAI блокирует, не тратим ресурсы. + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(new_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": len(discovered_urls) - len(new_urls), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_diff", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai") + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + with IAAIScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + batch_size = settings.celery.batch_size + for batch_start in range(0, len(discovered_urls), batch_size): + batch_urls = discovered_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(discovered_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": 0, + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_full_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": None, + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_rolling_refresh_sync( + *, + redis_client: Redis, + lane: str, + limit: int | None, + only_new: bool | None, + progress_callback=None, +) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai") + + existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:") + new_urls = [url for url in discovered_urls if url not in existing_urls] + + total_active = persistence.count_active_cars_for_lane("iaai:") + batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size)) + try: + offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0) + except Exception: + offset = 0 + + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix="iaai:", + offset=offset, + limit=batch_size, + ) + if not refresh_urls and total_active > 0: + offset = 0 + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix="iaai:", + offset=0, + limit=batch_size, + ) + + next_offset = 0 + if total_active > 0: + next_offset = offset + len(refresh_urls) + if next_offset >= total_active: + next_offset = 0 + + try: + redis_client.set(SITEMAP_HOURLY_REFRESH_OFFSET_KEY, str(next_offset)) + except Exception: + logger.warning("Failed to persist hourly rolling refresh offset", exc_info=True) + + seen: set[str] = set() + target_urls: list[str] = [] + for url in new_urls + refresh_urls: + if url in seen: + continue + seen.add(url) + target_urls.append(url) + + cars_upserted = 0 + cars_failed = 0 + protection_events = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if target_urls: + with IAAIScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) + worker_batch_size = settings.celery.batch_size + for batch_start in range(0, len(target_urls), worker_batch_size): + batch_urls = target_urls[batch_start:batch_start + worker_batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + worker_batch_size < len(target_urls): + time.sleep(_INTER_BATCH_DELAY) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": max(0, len(discovered_urls) - len(new_urls)), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_rolling_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "refresh_urls": len(refresh_urls), + "sold_marked": sold_count, + "protection_events": protection_events, + "transport": discovery_result.stats.transport, + "refresh_offset": offset, + "refresh_next_offset": next_offset, + "active_total": total_active, + } + + +def _start_stall_watchdog( + redis_client: Redis, + *, + task_id: str, + stall_timeout_seconds: int, + lock_key: str | None = None, + lock_owner: str | None = None, + db_idle_restart_seconds: int | None = None, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, stall_timeout_seconds / 3)) + + def _watchdog() -> None: + key = _task_progress_key(task_id) + no_data_count = 0 + # Абсолютный дедлайн: если watchdog работает дольше 3× stall_timeout без прогресса — убиваем. + watchdog_born = time.monotonic() + absolute_deadline = stall_timeout_seconds * 3 + while not stop_event.wait(interval_seconds): + db_idle_restart = False + try: + raw = redis_client.get(key) + if not raw: + no_data_count += 1 + elapsed_since_born = time.monotonic() - watchdog_born + if no_data_count % 5 == 0: + logger.warning( + "Stall watchdog: no progress data for task %s after %d checks (%.0fs)", + task_id, no_data_count, elapsed_since_born, + ) + # Если прогресс-данных нет дольше stall_timeout — считаем задачу мёртвой. + if elapsed_since_born > stall_timeout_seconds: + logger.error( + "Task %s has no progress data for %.0fs (> %ds); treating as stalled", + task_id, elapsed_since_born, stall_timeout_seconds, + ) + else: + continue + else: + no_data_count = 0 + data = json.loads(raw) + stage = data.get("stage") + last_ts = int(data.get("ts") or 0) + if not last_ts: + continue + db_idle_restart = bool( + db_idle_restart_seconds + and _should_restart_for_db_idle(data, db_idle_restart_seconds) + ) + if db_idle_restart: + logger.error( + "Task %s has no DB writes for >%ss at segment=%s/%s stage=%s; full restart required", + task_id, + db_idle_restart_seconds, + data.get("segment_index"), + data.get("segments_total"), + stage, + ) + else: + effective_stall_timeout = _stall_timeout_for_progress(stage, stall_timeout_seconds) + age = int(time.time()) - last_ts + if age < effective_stall_timeout: + watchdog_born = time.monotonic() # reset absolute deadline on real progress + continue + logger.error( + "Task %s stalled for %ss at stage=%s payload=%s; cleaning up and restarting", + task_id, + age, + stage, + data, + ) + except Exception: + logger.warning("Failed to inspect task progress for stall watchdog", exc_info=True) + # Если Redis тоже не отвечает дольше дедлайна — убиваем. + if time.monotonic() - watchdog_born > absolute_deadline: + logger.error("Stall watchdog: Redis unreachable for %.0fs; forcing kill", time.monotonic() - watchdog_born) + else: + continue + + if db_idle_restart: + _restart_bootstrap_from_first_segment( + redis_client, + reason=f"no DB writes for >{db_idle_restart_seconds}s", + ) + + # ── Pre-SIGTERM cleanup: release lock so next task can run ── + if lock_key and lock_owner: + try: + _release_lock_if_owner(redis_client, lock_key, lock_owner) + logger.info("Stall watchdog: released lock %s before SIGTERM", lock_key) + except Exception: + # Force-delete if owner check fails (process is dying anyway) + try: + redis_client.delete(lock_key) + logger.info("Stall watchdog: force-deleted lock %s", lock_key) + except Exception: + logger.warning("Stall watchdog: failed to release lock %s", lock_key, exc_info=True) + + # ── Queue a followup task so parsing resumes after restart ── + try: + followup_ttl = max(180, int(stall_timeout_seconds) + 300) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + from ..worker.celery_app import celery_app + celery_app.send_task( + SYNC_LISTING_TASK_NAME, + kwargs={}, + queue=IAAI_SYNC_QUEUE, + countdown=15, + expires=followup_ttl, + ) + logger.info("Stall watchdog: queued followup sync_listing_task after stall") + else: + logger.info("Stall watchdog: followup already pending, skip duplicate enqueue") + except Exception: + try: + _clear_followup_pending(redis_client) + except Exception: + pass + logger.warning("Stall watchdog: failed to queue followup task", exc_info=True) + + # SIGTERM даёт процессу время на cleanup (закрыть DB, browser). + # Celery перехватит SIGTERM и поднимет Terminated / warm shutdown. + try: + os.kill(os.getpid(), signal.SIGTERM) + except OSError: + pass + # Даём 30 секунд на graceful shutdown, потом SIGKILL как последний resort. + stop_event.wait(30) + if not stop_event.is_set(): + logger.error("Task %s did not stop after SIGTERM; forcing SIGKILL", task_id) + os.kill(os.getpid(), signal.SIGKILL) + + thread = Thread(target=_watchdog, name=f"task-stall-watchdog-{task_id[:8]}", daemon=True) + thread.start() + return stop_event, thread + + +def _should_restart_for_db_idle(progress: dict, db_idle_restart_seconds: int) -> bool: + stage = str(progress.get("stage") or "") + if stage in TERMINAL_PROGRESS_STAGES: + return False + if stage in STALL_WATCHDOG_LONG_RUNNING_STAGES: + timeout = _stall_timeout_for_progress(stage, db_idle_restart_seconds) + progress_ts = _safe_int(progress.get("ts")) or 0 + return progress_ts > 0 and int(time.time()) - progress_ts >= timeout + + segments_total = _safe_int(progress.get("segments_total")) + segment_index = _safe_int(progress.get("segment_index")) + if segments_total is None or segment_index is None: + return False + if segments_total <= 0 or segment_index >= segments_total - 1: + return False + + now_ts = int(time.time()) + progress_ts = _safe_int(progress.get("ts")) or 0 + if progress_ts <= 0: + return False + + db_progress_ts = _safe_int(progress.get("last_db_progress_ts")) + if db_progress_ts is None: + db_progress_ts = _read_global_db_progress_ts() + task_started_ts = _safe_int(progress.get("task_started_ts")) or progress_ts + last_db_or_start_ts = max(db_progress_ts or 0, task_started_ts) + return now_ts - last_db_or_start_ts >= int(db_idle_restart_seconds) + + +def _safe_int(value) -> int | None: + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _mobilede_should_skip_dynamic_segment(total_results: int | None) -> bool: + return MOBILEDE_DYNAMIC_SEGMENT_PROBES and MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS and total_results == 0 + + +def _read_global_db_progress_ts() -> int | None: + try: + redis_client = _get_redis() + raw = redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) + return _safe_int(raw) + except Exception: + return None + + +def _mobilede_segment_key(segment: dict[str, object] | None) -> str: + if not segment: + return "all" + listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if listing_url: + digest = hashlib.sha1(listing_url.encode("utf-8")).hexdigest()[:16] + return f"url:{digest}" + make_id = str(segment.get("make_id") or segment.get("makeId") or segment.get("make") or "all").strip() + model_id = str(segment.get("model_id") or segment.get("modelId") or segment.get("model") or "all").strip() + return f"{make_id}:{model_id}".replace(" ", "_") + + +def _mobilede_segment_fingerprint(segment: dict[str, object] | None) -> str: + if not segment: + return "all" + payload = json.dumps(segment, ensure_ascii=False, sort_keys=True, default=str) + return hashlib.sha1(payload.encode("utf-8")).hexdigest()[:16] + + +def _mobilede_cursor_key(segment: dict[str, object] | None) -> str: + if not segment: + return MOBILEDE_SEARCH_CURSOR_KEY + return MOBILEDE_SEGMENT_CURSOR_KEY_FMT.format(segment_key=_mobilede_segment_fingerprint(segment)) + + +def _mobilede_progress_page_counter_key(segment: dict[str, object] | None) -> str: + return MOBILEDE_PROGRESS_PAGE_COUNTER_KEY_FMT.format(segment_key=_mobilede_segment_fingerprint(segment)) + + +def _mobilede_segment_zero_insert_streak_key(segment: dict[str, object] | None) -> str: + return f"mobilede:state:segment_zero_insert_streak:{_mobilede_segment_fingerprint(segment)}" + + +def _mobilede_segment_cooldown_key(segment: dict[str, object] | None) -> str: + return f"mobilede:state:segment_cooldown:{_mobilede_segment_fingerprint(segment)}" + + +def _mobilede_segment_hot_key(segment: dict[str, object] | None) -> str: + return f"mobilede:state:segment_hot:{_mobilede_segment_fingerprint(segment)}" + + +def _mobilede_segment_in_cooldown(redis_client: Redis, segment: dict[str, object] | None) -> bool: + if not segment: + return False + return bool(redis_client.ttl(_mobilede_segment_cooldown_key(segment)) > 0) + + +def _mobilede_segment_is_hot(redis_client: Redis, segment: dict[str, object] | None) -> bool: + if not segment: + return False + return bool(redis_client.ttl(_mobilede_segment_hot_key(segment)) > 0) + + +def _mobilede_has_hot_segments(redis_client: Redis, segments: list[dict[str, object]]) -> bool: + for segment in segments: + if _mobilede_segment_is_hot(redis_client, segment): + return True + return False + + +def _mobilede_update_segment_freshness_state( + redis_client: Redis, + *, + segment: dict[str, object] | None, + only_new: bool | None, + inserted: int, + listings: int, +) -> None: + if not segment or only_new is not True: + return + streak_key = _mobilede_segment_zero_insert_streak_key(segment) + cooldown_key = _mobilede_segment_cooldown_key(segment) + hot_key = _mobilede_segment_hot_key(segment) + listings_count = max(0, int(listings)) + if inserted > 0: + ratio = (float(inserted) / float(listings_count)) if listings_count > 0 else 0.0 + if ratio >= MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO: + redis_client.delete(streak_key) + redis_client.delete(cooldown_key) + redis_client.set(hot_key, "1", ex=MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS) + return + # Есть новые, но доля слишком низкая: сегмент считается холодным для only_new режима. + redis_client.delete(hot_key) + redis_client.set(cooldown_key, "1", ex=MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS) + logger.info( + "mobile.de segment marked cold by ratio: segment=%s inserted=%s listings=%s ratio=%.3f threshold=%.3f cooldown=%ss", + _mobilede_segment_label(segment), + inserted, + listings_count, + ratio, + MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO, + MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS, + ) + return + streak = int(redis_client.incr(streak_key)) + redis_client.expire(streak_key, 24 * 60 * 60) + if streak >= MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK: + redis_client.set(cooldown_key, "1", ex=MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS) + logger.info( + "mobile.de segment cooldown enabled: segment=%s streak=%s cooldown=%ss", + _mobilede_segment_label(segment), + streak, + MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS, + ) + + +def _mobilede_segment_label(segment: dict[str, object] | None) -> str: + if not segment: + return "all" + label = str(segment.get("label") or "").strip() + return label or _mobilede_segment_key(segment) + + +def _find_mobilede_runtime_segment( + settings: Settings, + *, + search_url: str | None = None, + make_id: str | None, + model_id: str | None, +) -> dict[str, object] | None: + target_search_url = str(search_url or "").strip() + target_make_id = str(make_id or "").strip() + target_model_id = str(model_id or "").strip() + if not target_search_url and not target_make_id and not target_model_id: + return None + for candidate in _build_mobilede_runtime_segments(settings): + candidate_search_url = str(candidate.get("search_url") or candidate.get("listing_url") or "").strip() + if target_search_url and candidate_search_url == target_search_url: + return candidate + candidate_make_id = str(candidate.get("make_id") or "").strip() + candidate_model_id = str(candidate.get("model_id") or "").strip() + if candidate_make_id == target_make_id and candidate_model_id == target_model_id: + return candidate + return None + + +def _mobilede_segment_make(segment: dict[str, object] | None, fallback: str | None = None) -> str: + if segment: + listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if listing_url: + return "filtered-url" + value = str(segment.get("make") or "").strip() + if value: + return value + return str(fallback or "all").strip() or "all" + + +def _mobilede_segment_model(segment: dict[str, object] | None, fallback: str | None = None) -> str: + if segment: + listing_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if listing_url: + return "filtered-url" + value = str(segment.get("model") or "").strip() + if value: + return value + return str(fallback or "all").strip() or "all" + + +def _mobilede_segment_uses_url(segment: dict[str, object] | None, search_url: str | None = None) -> bool: + if search_url and str(search_url).strip(): + return True + if not segment: + return False + return bool(str(segment.get("search_url") or segment.get("listing_url") or "").strip()) + + +def _mobilede_filter_source(segment: dict[str, object] | None, search_url: str | None = None) -> str: + return "search_url" if _mobilede_segment_uses_url(segment, search_url) else "params" + + +def _is_mobilede_transient_request_error(exc: Exception) -> bool: + if isinstance( + exc, + ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ProxyError, + requests.exceptions.SSLError, + ), + ): + return True + text = str(exc).lower() + return any( + marker in text + for marker in ( + "nameresolutionerror", + "temporary failure in name resolution", + "max retries exceeded", + "connection refused", + "read timed out", + "connect timeout", + ) + ) + + +def _mobilede_task_result_summary( + *, + result: dict[str, object], + segment: dict[str, object] | None, + start_page: int, + end_page: int, + make_name: str, + model_name: str, +) -> dict[str, object]: + upsert = result.get("upsert") if isinstance(result.get("upsert"), dict) else {} + return { + "status": "success", + "source": "mobile.de", + "run_id": result.get("run_id"), + "segment": _mobilede_segment_label(segment), + "make": make_name, + "model": model_name, + "pages": { + "start": start_page, + "end": end_page, + "count": end_page - start_page + 1, + }, + "listing_count": int(result.get("listing_count", 0) or 0), + "unique_listing_count": int(result.get("unique_listing_count", 0) or 0), + "upsert": { + "inserted": int(upsert.get("inserted", 0) or 0), + "updated": int(upsert.get("updated", 0) or 0), + "images_upserted": int(upsert.get("images_upserted", 0) or 0), + }, + } + + +def _log_mobilede_progress_threshold( + redis_client: Redis, + *, + task_id: str, + segment: dict[str, object] | None, + delta_pages: int, + delta_cars: int, + delta_images: int, + start_page: int, + end_page: int, +) -> None: + if delta_pages <= 0: + return + try: + counter_key = _mobilede_progress_page_counter_key(segment) + total_pages = int(redis_client.incrby(counter_key, int(delta_pages))) + redis_client.expire(counter_key, 7 * 24 * 60 * 60) + previous_total = total_pages - int(delta_pages) + if previous_total // MOBILEDE_PROGRESS_LOG_EVERY_PAGES == total_pages // MOBILEDE_PROGRESS_LOG_EVERY_PAGES: + return + logger.info( + "mobile.de progress: segment=%s total_pages=%s delta_pages=%s cars=%s images=%s last_window=%s-%s task_id=%s", + _mobilede_segment_label(segment), + total_pages, + delta_pages, + delta_cars, + delta_images, + start_page, + end_page, + task_id, + ) + except Exception: + logger.debug("Failed to update mobile.de aggregated progress", exc_info=True) + + +def _mobilede_url_query_value(search_url: str, key: str) -> str | None: + for item_key, item_value in parse_qsl(urlsplit(search_url).query, keep_blank_values=True): + if item_key == key and item_value != "": + return item_value + return None + + +def _mobilede_make_segment_url(search_url: str, **params: str | int | None) -> str: + return MobileDeClient.build_search_url_from_existing(search_url, page_number=1, **params) + + +def _mobilede_apply_newest_sort_to_url(search_url: str | None) -> str | None: + if not search_url: + return search_url + return MobileDeClient.build_search_url_from_existing(search_url, page_number=1, sb="doc", od="down") + + +def _mobilede_is_strict_first_pass_mode(redis_client: Redis, *, segment: dict[str, object] | None, only_new: bool | None) -> bool: + return bool( + MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS + and only_new is True + and segment is not None + and _mobilede_bootstrap_done(redis_client) + and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP + ) + + +def _mobilede_reserve_incremental_cycle(redis_client: Redis, *, total_segments: int | None) -> tuple[str, bool]: + total = max(1, int(total_segments or 1)) + cycle = str(redis_client.get(MOBILEDE_INCREMENTAL_CYCLE_KEY) or "1") + seen = int(redis_client.incr(MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY)) + if seen >= total: + next_cycle = str(int(cycle) + 1) + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_KEY, next_cycle) + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY, "0") + return cycle, seen == 1 + + +def _mobilede_cycle_cursor_key(base_cursor_key: str, cycle_id: str) -> str: + return f"{base_cursor_key}:cycle:{cycle_id}" + + +def _mobilede_cycle_seen_set_key(cycle_id: str) -> str: + return MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT.format(cycle_id=cycle_id) + + +def _mobilede_try_mark_cycle_segment_seen( + redis_client: Redis, + *, + cycle_id: str, + segment: dict[str, object] | None, + ttl_seconds: int = 24 * 60 * 60, +) -> bool: + fingerprint = _mobilede_segment_fingerprint(segment) + set_key = _mobilede_cycle_seen_set_key(cycle_id) + added = int(redis_client.sadd(set_key, fingerprint)) + redis_client.expire(set_key, ttl_seconds) + return added == 1 + + +def _mobilede_current_cycle_id(redis_client: Redis) -> str: + cycle_id = str(redis_client.get(MOBILEDE_INCREMENTAL_CYCLE_KEY) or "").strip() + if not cycle_id: + cycle_id = "1" + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_KEY, cycle_id) + return cycle_id + + +def _mobilede_reserve_strict_first_pass_segment( + redis_client: Redis, + settings: Settings, + *, + segment: dict[str, object] | None, + segment_index: int | None, +) -> tuple[dict[str, object] | None, int | None, str]: + segments = _get_cached_mobilede_runtime_segments(redis_client) or [] + total_segments = len(segments) + cycle_id = _mobilede_current_cycle_id(redis_client) + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + only_new = runtime_config.sync.only_new + + def _try_take_current() -> bool: + return segment is not None and _mobilede_try_mark_cycle_segment_seen( + redis_client, + cycle_id=cycle_id, + segment=segment, + ) + + if _try_take_current(): + return segment, segment_index, cycle_id + + for _ in range(max(1, total_segments)): + reservation = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reservation is None: + break + next_index, next_segment = reservation + if _mobilede_try_mark_cycle_segment_seen( + redis_client, + cycle_id=cycle_id, + segment=next_segment, + ): + return next_segment, next_index, cycle_id + + # Все сегменты цикла пройдены: начинаем новый цикл и берём первый доступный. + cycle_id = str(int(cycle_id) + 1) + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_KEY, cycle_id) + redis_client.set(MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY, "0") + + if segment is not None and _mobilede_try_mark_cycle_segment_seen( + redis_client, + cycle_id=cycle_id, + segment=segment, + ): + return segment, segment_index, cycle_id + + for _ in range(max(1, total_segments)): + reservation = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reservation is None: + break + next_index, next_segment = reservation + if _mobilede_try_mark_cycle_segment_seen( + redis_client, + cycle_id=cycle_id, + segment=next_segment, + ): + return next_segment, next_index, cycle_id + + return segment, segment_index, cycle_id + + +def _mobilede_price_ranges() -> list[tuple[int, int | None]]: + raw_ranges = os.getenv("MOBILEDE_PRICE_RANGES", "").strip() + if raw_ranges: + parsed: list[tuple[int, int | None]] = [] + for raw_item in raw_ranges.split(","): + item = raw_item.strip() + if not item: + continue + left, _, right = item.partition(":") + try: + min_value = int(left.strip()) if left.strip() else 1 + max_value = int(right.strip()) if right.strip() else None + parsed.append((min_value, max_value)) + except ValueError: + logger.warning("Invalid MOBILEDE_PRICE_RANGES item ignored: %s", item) + if parsed: + return parsed + compact = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} + if compact: + return [ + (1, 5000), + (5001, 10000), + (10001, 15000), + (15001, 20000), + (20001, 30000), + (30001, 50000), + (50001, 75000), + (75001, 100000), + (100001, 150000), + (150001, None), + ] + return [(1, 500), (500, 1000), (1001, 1500), (1501, 2000), (2001, 2500), (2501, 3000), (3001, 4000), (4001, 5000), (5001, 7500), (7501, 10000), (10001, 12500), (12501, 15000), (15001, 17500), (17501, 20000), (20001, 25000), (25001, 30000), (30001, 40000), (40001, 50000), (50001, 75000), (75001, 100000), (100001, 150000), (150001, None)] + + +def _mobilede_year_ranges() -> list[tuple[int | None, int | None]]: + compact = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} + if compact: + return [(None, 2009), (2010, 2017), (2018, 2022), (2023, None)] + return [(None, 1999), (2000, 2004), (2005, 2009), (2010, 2014), (2015, 2017), (2018, 2020), (2021, 2022), (2023, 2024), (2025, None)] + + +def _mobilede_mileage_ranges() -> list[tuple[int | None, int | None]]: + compact = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"} + if compact: + return [(None, 100000), (100001, 200000), (200001, None)] + return [(None, 50000), (50001, 100000), (100001, 150000), (150001, 200000), (200001, None)] + + +def _mobilede_range_value(min_value: int | None, max_value: int | None) -> str: + return f"{min_value or ''}:{max_value or ''}" + + +def _mobilede_price_label(price_min: int, price_max: int | None) -> str: + return f"price={price_min}-{price_max}" if price_max is not None else f"price={price_min}+" + + +def _mobilede_year_label(year_min: int | None, year_max: int | None) -> str: + if year_min is None: + return f"year<={year_max}" + if year_max is None: + return f"year>={year_min}" + return f"year={year_min}-{year_max}" + + +def _mobilede_mileage_label(mileage_min: int | None, mileage_max: int | None) -> str: + if mileage_min is None: + return f"km<={mileage_max}" + if mileage_max is None: + return f"km>={mileage_min}" + return f"km={mileage_min}-{mileage_max}" + + +def _mobilede_probe_total(search_url: str, **params: str | int | None) -> int | None: + try: + client = MobileDeClient.for_worker(delay_seconds=0) + page = client.fetch_search_page(page_number=1, search_url=search_url, **params) + return int(page.total_results or 0) + except Exception as exc: + logger.warning("mobile.de segment probe failed: params=%s error=%s", params, exc) + return None + + +def _mobilede_segment_pages_for_total(total_results: int | None, fallback_max_pages: int) -> int: + if total_results is None or total_results <= 0: + return min(fallback_max_pages, 100) + pages = max(1, min(100, (int(total_results) + 19) // 20)) + return min(fallback_max_pages, pages) + + +def _mobilede_make_expanded_segment( + base_segment: dict[str, object], + *, + search_url: str, + base_label: str, + label_parts: list[str], + params: dict[str, str | int | None], + total_results: int | None, + fallback_max_pages: int, +) -> dict[str, object]: + item = dict(base_segment) + item["search_url"] = _mobilede_make_segment_url(search_url, **params) + item["listing_url"] = item["search_url"] + item["start_page"] = 1 + item["max_pages"] = _mobilede_segment_pages_for_total(total_results, fallback_max_pages) + item["total_results"] = total_results + item["label"] = f"{base_label} | {' | '.join(label_parts)} | total~{total_results if total_results is not None else '?'}" + return item + + +def _expand_mobilede_search_url_segment(segment: dict[str, object]) -> list[dict[str, object]]: + search_url = str(segment.get("search_url") or segment.get("listing_url") or "").strip() + if not search_url: + return [segment] + # Если пользователь уже задал узкий price/year/mileage range — не размножаем автоматически. + existing_range_keys = {"p", "fr", "ml"} + query_keys = {key for key, _ in parse_qsl(urlsplit(search_url).query, keep_blank_values=True)} + if query_keys & existing_range_keys: + return [segment] + + expanded: list[dict[str, object]] = [] + base_label = str(segment.get("label") or "mobile.de segmented URL") + max_pages = int(segment.get("max_pages") or 100) + for price_min, price_max in _mobilede_price_ranges(): + params: dict[str, str | int | None] = {} + if price_max is not None: + params["p"] = f"{price_min}:{price_max}" + else: + params["p"] = f"{price_min}:" + price_label = _mobilede_price_label(price_min, price_max) + price_total = _mobilede_probe_total(search_url, **params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None + if _mobilede_should_skip_dynamic_segment(price_total): + continue + if price_total is not None and price_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[price_label], + params=params, + total_results=price_total, + fallback_max_pages=max_pages, + ) + item["price_min"] = str(price_min) + item["price_max"] = str(price_max) if price_max is not None else None + expanded.append(item) + continue + + for year_min, year_max in _mobilede_year_ranges(): + year_params = dict(params) + year_params["fr"] = _mobilede_range_value(year_min, year_max) + year_label = _mobilede_year_label(year_min, year_max) + year_total = _mobilede_probe_total(search_url, **year_params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None + if _mobilede_should_skip_dynamic_segment(year_total): + continue + if year_total is not None and year_total <= MOBILEDE_SEGMENT_TARGET_RESULTS: + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[price_label, year_label], + params=year_params, + total_results=year_total, + fallback_max_pages=max_pages, + ) + item["price_min"] = str(price_min) + item["price_max"] = str(price_max) if price_max is not None else None + item["year_min"] = str(year_min) if year_min is not None else None + item["year_max"] = str(year_max) if year_max is not None else None + expanded.append(item) + continue + + for mileage_min, mileage_max in _mobilede_mileage_ranges(): + mileage_params = dict(year_params) + mileage_params["ml"] = _mobilede_range_value(mileage_min, mileage_max) + mileage_label = _mobilede_mileage_label(mileage_min, mileage_max) + mileage_total = _mobilede_probe_total(search_url, **mileage_params) if MOBILEDE_DYNAMIC_SEGMENT_PROBES else None + if _mobilede_should_skip_dynamic_segment(mileage_total): + continue + item = _mobilede_make_expanded_segment( + segment, + search_url=search_url, + base_label=base_label, + label_parts=[price_label, year_label, mileage_label], + params=mileage_params, + total_results=mileage_total, + fallback_max_pages=max_pages, + ) + item["price_min"] = str(price_min) + item["price_max"] = str(price_max) if price_max is not None else None + item["year_min"] = str(year_min) if year_min is not None else None + item["year_max"] = str(year_max) if year_max is not None else None + item["mileage_min"] = str(mileage_min) if mileage_min is not None else None + item["mileage_max"] = str(mileage_max) if mileage_max is not None else None + expanded.append(item) + return expanded + + +def _build_mobilede_runtime_segments(settings: Settings) -> list[dict[str, object]]: + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + segments = [segment.to_task_kwargs() for segment in runtime_config.mobilede.segments] + expanded: list[dict[str, object]] = [] + for segment in segments: + if segment.get("auto_segment") is False: + expanded.append(segment) + continue + expanded.extend(_expand_mobilede_search_url_segment(segment)) + if len(expanded) != len(segments): + logger.info("mobile.de runtime segments expanded: %s -> %s", len(segments), len(expanded)) + return expanded + + +def _get_mobilede_runtime_segments(redis_client: Redis, settings: Settings) -> list[dict[str, object]]: + try: + cached_raw = redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY) + if cached_raw: + cached = json.loads(cached_raw) + if isinstance(cached, list): + return [dict(item) for item in cached if isinstance(item, dict)] + except Exception: + logger.debug("Failed to read mobile.de runtime segment cache", exc_info=True) + + segments = _build_mobilede_runtime_segments(settings) + try: + redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY, json.dumps(segments, ensure_ascii=False), ex=24 * 60 * 60) + except Exception: + logger.debug("Failed to write mobile.de runtime segment cache", exc_info=True) + return segments + + +def _get_cached_mobilede_runtime_segments(redis_client: Redis) -> list[dict[str, object]] | None: + try: + cached_raw = redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY) + if not cached_raw: + return None + cached = json.loads(cached_raw) + if isinstance(cached, list): + return [dict(item) for item in cached if isinstance(item, dict)] + except Exception: + logger.debug("Failed to read cached mobile.de runtime segments", exc_info=True) + return None + + +def _request_mobilede_runtime_segments_rebuild(redis_client: Redis) -> None: + try: + redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY, "1", ex=15 * 60) + except Exception: + logger.debug("Failed to request mobile.de runtime segment rebuild", exc_info=True) + + +def _mobilede_bootstrap_done(redis_client: Redis) -> bool: + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: + return True + return bool(redis_client.get(MOBILEDE_BOOTSTRAP_DONE_KEY)) + + +def _mobilede_segment_scan_complete(redis_client: Redis, segment: dict[str, object] | None) -> bool: + if not segment: + return False + cursor_raw = redis_client.get(_mobilede_cursor_key(segment)) + segment_max_pages = int(segment.get("max_pages") or 100) + return cursor_raw is not None and int(cursor_raw) >= segment_max_pages + + +def _mobilede_bootstrap_segment_done(redis_client: Redis, segment: dict[str, object] | None) -> bool: + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or not segment: + return False + return bool(redis_client.get(f"mobilede:state:bootstrap_segment_done:{_mobilede_segment_fingerprint(segment)}")) + + +def _mark_mobilede_bootstrap_segment_done(redis_client: Redis, segment: dict[str, object] | None, total_segments: int | None) -> None: + if not MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED or not segment: + return + segment_done_key = f"mobilede:state:bootstrap_segment_done:{_mobilede_segment_fingerprint(segment)}" + try: + if total_segments is not None: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(int(total_segments))) + if redis_client.setnx(segment_done_key, "1"): + redis_client.expire(segment_done_key, 30 * 24 * 60 * 60) + done = int(redis_client.incr(MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY)) + total = int(redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) or total_segments or 0) + logger.info( + "mobile.de bootstrap segment completed: %s done=%s/%s", + _mobilede_segment_label(segment), + done, + total, + ) + if total > 0 and done >= total: + redis_client.set(MOBILEDE_BOOTSTRAP_DONE_KEY, "1") + logger.info("mobile.de bootstrap full scan completed: segments=%s", total) + except Exception: + logger.debug("Failed to mark mobile.de bootstrap segment complete", exc_info=True) + + +def _reserve_next_mobilede_runtime_segment( + redis_client: Redis, + settings: Settings, + *, + only_new: bool | None = None, +) -> tuple[int, dict[str, object]] | None: + segments = _get_cached_mobilede_runtime_segments(redis_client) + if not segments: + _request_mobilede_runtime_segments_rebuild(redis_client) + segments = _build_mobilede_runtime_segments(settings) if not MOBILEDE_DYNAMIC_SEGMENT_PROBES else [] + if not segments: + return None + hot_only_active = bool(MOBILEDE_ONLY_NEW_HOT_ONLY and only_new is True) + has_hot_segments = _mobilede_has_hot_segments(redis_client, segments) if hot_only_active else False + skip_completed_bootstrap = MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED and not _mobilede_bootstrap_done(redis_client) + + def _try_reserve(*, require_hot: bool, respect_cooldown: bool) -> tuple[int, dict[str, object]] | None: + for _ in range(len(segments)): + next_index = int(redis_client.incr(MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY)) - 1 + segment_index = next_index % len(segments) + segment = segments[segment_index] + if skip_completed_bootstrap and _mobilede_bootstrap_segment_done(redis_client, segment): + continue + if respect_cooldown and _mobilede_segment_in_cooldown(redis_client, segment): + continue + if require_hot and hot_only_active and has_hot_segments and not _mobilede_segment_is_hot(redis_client, segment): + continue + return segment_index + 1, segment + return None + + # 1) Обычный hot-only проход. + reserved = _try_reserve(require_hot=True, respect_cooldown=True) + if reserved is not None: + return reserved + # 2) Если hot-only выжег очередь, разрешаем любые не-cooldown сегменты. + reserved = _try_reserve(require_hot=False, respect_cooldown=True) + if reserved is not None: + return reserved + # 3) Failsafe: если всё в cooldown, берём любой сегмент (иначе парсер встанет). + reserved = _try_reserve(require_hot=False, respect_cooldown=False) + if reserved is not None: + return reserved + return None + + +def _enqueue_mobilede_runtime_segments( + *, + lane: str, + delay_seconds: float, + use_cursor: bool, + continuous: bool, +) -> list[dict[str, object]]: + settings = Settings() + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + only_new = runtime_config.sync.only_new + redis_client = _get_redis() + segments = _get_cached_mobilede_runtime_segments(redis_client) + if not segments: + _request_mobilede_runtime_segments_rebuild(redis_client) + segments = _build_mobilede_runtime_segments(settings) if not MOBILEDE_DYNAMIC_SEGMENT_PROBES else [] + if not segments: + return [] + try: + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(segments))) + except Exception: + logger.debug("Failed to initialize mobile.de bootstrap segment total", exc_info=True) + initial_reservations: list[tuple[int, dict[str, object]]] = [] + for _ in range(min(MOBILEDE_RUNTIME_INITIAL_TASKS, len(segments))): + reservation = _reserve_next_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reservation is None: + break + initial_reservations.append(reservation) + for index, segment in initial_reservations: + mobilede_sync_search_task.apply_async( + kwargs={ + "start_page": int(segment.get("start_page") or 1), + "max_pages": int(segment.get("max_pages") or 5), + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": continuous, + "segment": segment, + "segment_index": index, + "runtime_rotation": True, + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + return segments + + +def _reserve_mobilede_runtime_segment( + redis_client: Redis, + settings: Settings, + *, + only_new: bool | None = None, +) -> tuple[int, dict[str, object]] | None: + segments = _get_cached_mobilede_runtime_segments(redis_client) + if not segments: + _request_mobilede_runtime_segments_rebuild(redis_client) + segments = _build_mobilede_runtime_segments(settings) if not MOBILEDE_DYNAMIC_SEGMENT_PROBES else [] + if not segments: + return None + hot_only_active = bool(MOBILEDE_ONLY_NEW_HOT_ONLY and only_new is True) + has_hot_segments = _mobilede_has_hot_segments(redis_client, segments) if hot_only_active else False + skip_completed_bootstrap = MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED and not _mobilede_bootstrap_done(redis_client) + + def _try_reserve(*, require_hot: bool, respect_cooldown: bool) -> tuple[int, dict[str, object]] | None: + for _ in range(len(segments)): + next_index = int(redis_client.incr(MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY)) - 1 + segment_index = next_index % len(segments) + segment = segments[segment_index] + if skip_completed_bootstrap and _mobilede_bootstrap_segment_done(redis_client, segment): + continue + if respect_cooldown and _mobilede_segment_in_cooldown(redis_client, segment): + continue + if require_hot and hot_only_active and has_hot_segments and not _mobilede_segment_is_hot(redis_client, segment): + continue + return segment_index, segment + return None + + reserved = _try_reserve(require_hot=True, respect_cooldown=True) + if reserved is not None: + return reserved + reserved = _try_reserve(require_hot=False, respect_cooldown=True) + if reserved is not None: + return reserved + reserved = _try_reserve(require_hot=False, respect_cooldown=False) + if reserved is not None: + return reserved + return None + + +def _reserve_mobilede_page_window( + redis_client: Redis, + *, + requested_start_page: int, + page_window_size: int, + use_cursor: bool, + cursor_key: str, +) -> tuple[int, int]: + page_window_size = max(1, int(page_window_size)) + requested_start_page = max(1, int(requested_start_page)) + if not use_cursor: + return requested_start_page, requested_start_page + page_window_size - 1 + redis_client.setnx(cursor_key, str(requested_start_page - 1)) + window_end = int(redis_client.incrby(cursor_key, page_window_size)) + window_start = max(1, window_end - page_window_size + 1) + return window_start, window_end + + +def _reset_mobilede_page_cursor(redis_client: Redis, *, cursor_key: str, next_start_page: int = 1) -> None: + redis_client.set(cursor_key, str(max(0, int(next_start_page) - 1))) + + +_persistence_instance: PersistenceService | None = None + + +def _get_persistence() -> PersistenceService: + global _persistence_instance + if _persistence_instance is not None: + return _persistence_instance + + sett = Settings() + persistence = PersistenceService(sett) + + def _ping_db() -> None: + with persistence.engine.connect() as conn: + conn.exec_driver_sql("SELECT 1") + + _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0) + _persistence_instance = persistence + return persistence + + +_redis_instance: Redis | None = None + + +def _get_redis() -> Redis: + global _redis_instance + if _redis_instance is not None: + try: + _redis_instance.ping() + return _redis_instance + except Exception: + _redis_instance = None + + sett = Settings() + redis_client = Redis.from_url( + sett.redis.url, + decode_responses=True, + socket_connect_timeout=sett.redis.socket_connect_timeout_seconds, + socket_timeout=sett.redis.socket_timeout_seconds, + health_check_interval=sett.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + def _ping_redis() -> None: + redis_client.ping() + + _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0) + _redis_instance = redis_client + return redis_client + + +def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: + try: + acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + if acquired: + return True + + # Автовосстановление: если lock завис без TTL, считаем stale и пересоздаём. + ttl = redis_client.ttl(key) + if ttl is not None and ttl < 0: + logger.warning("Detected stale lock without TTL, removing: %s", key) + redis_client.delete(key) + return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + + return False + except Exception as exc: + logger.warning("Failed to acquire lock %s", key, exc_info=True) + return False + + +def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: + try: + refreshed = redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) + end + return 0 + """, + 1, + key, + owner_token, + int(ttl_seconds), + ) + return bool(refreshed) + except Exception as exc: + logger.warning("Failed to refresh lock %s", key, exc_info=True) + return None + + +def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: + try: + redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) + end + return 0 + """, + 1, + key, + owner_token, + ) + except Exception as exc: + logger.warning("Failed to release lock %s", key, exc_info=True) + + +def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool: + try: + inspector = celery_app.control.inspect(timeout=1.0) + snapshots = [ + inspector.active() or {}, + inspector.reserved() or {}, + inspector.scheduled() or {}, + ] + except Exception: + logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True) + return True + + for snapshot in snapshots: + for entries in snapshot.values(): + for entry in entries or []: + task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") + if task_name != SYNC_LISTING_TASK_NAME: + continue + # Исключаем текущую задачу — она не считается "другой запущенной" + entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "") + if exclude_task_id and entry_id == exclude_task_id: + continue + return True + return False + + +def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool: + try: + owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY) + if not owner_token: + return False + except Exception: + logger.warning("Failed to read sync listing lock before cleanup", exc_info=True) + return False + + if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id): + logger.info("sync_listing lock preserved: active task still detected") + return False + + try: + ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY) + redis_client.delete(SYNC_LISTING_LOCK_KEY) + logger.warning( + "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart", + owner_token, + ttl, + ) + return True + except Exception: + logger.warning("Failed to clear orphan sync listing lock", exc_info=True) + return False + + +def _is_full_scan_done(redis_client: Redis) -> bool: + try: + value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY) + except Exception: + logger.warning("Failed to read full scan state", exc_info=True) + return False + return str(value or "").strip() == "1" + + +def _set_full_scan_done(redis_client: Redis, done: bool) -> None: + try: + redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0") + except Exception: + logger.warning("Failed to persist full scan state", exc_info=True) + + +def _load_last_completed_segment(redis_client: Redis) -> int | None: + # Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. + try: + raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY) + except Exception: + logger.warning("Failed to read sync listing checkpoint", exc_info=True) + return None + if raw is None: + return None + try: + value = int(str(raw).strip()) + except (TypeError, ValueError): + logger.warning("Invalid sync listing checkpoint value %r; clearing", raw) + _clear_sync_checkpoint(redis_client) + return None + if value < 0: + _clear_sync_checkpoint(redis_client) + return None + return value + + +def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None: + try: + redis_client.set( + SYNC_LISTING_CHECKPOINT_KEY, + str(int(segment_index)), + ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to save sync listing checkpoint", exc_info=True) + + +def _restart_bootstrap_from_first_segment(redis_client: Redis, *, reason: str) -> None: + """Clear bootstrap checkpoint so the next full scan starts from segment 1.""" + try: + pipe = redis_client.pipeline() + pipe.delete(SYNC_LISTING_CHECKPOINT_KEY) + pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) + pipe.delete(GLOBAL_PROGRESS_TS_KEY) + pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY) + pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0") + pipe.execute() + logger.error("Bootstrap restart requested from segment 1: %s", reason) + except Exception: + logger.warning("Failed to reset bootstrap checkpoint for DB-idle restart", exc_info=True) + + +def _clear_sync_checkpoint(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY) + except Exception: + logger.warning("Failed to clear sync listing checkpoint", exc_info=True) + + +def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool: + try: + return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds)))) + except Exception: + logger.warning("Failed to set follow-up pending flag", exc_info=True) + return True + + +def _clear_followup_pending(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) + except Exception: + logger.warning("Failed to clear follow-up pending flag", exc_info=True) + + +def _bump_bootstrap_failure_streak( + redis_client: Redis, + *, + reason: str, +) -> tuple[int, bool]: + try: + streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)) + redis_client.expire( + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to update bootstrap failure streak", exc_info=True) + return 0, True + + should_enqueue = streak < SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT + if should_enqueue: + logger.warning( + "Bootstrap failure streak %d/%d recorded (reason=%s)", + streak, + SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, + reason, + ) + else: + logger.error( + "Bootstrap follow-up circuit breaker opened after %d consecutive failures (reason=%s)", + streak, + reason, + ) + return streak, should_enqueue + + +def _clear_bootstrap_failure_streak(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY) + except Exception: + logger.warning("Failed to clear bootstrap failure streak", exc_info=True) + + +def _bump_bootstrap_continuation_streak(redis_client: Redis) -> tuple[int, bool]: + """Счётчик подряд идущих bootstrap-followup запусков. + + Возвращает (streak, should_continue). Если should_continue=False, + немедленные continuation блокируются до следующего beat-цикла. + """ + try: + streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY)) + redis_client.expire( + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to update bootstrap continuation streak", exc_info=True) + return 0, True + + should_continue = streak <= SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT + if not should_continue: + logger.error( + "Bootstrap continuation breaker OPEN: streak=%d limit=%d", + streak, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT, + ) + return streak, should_continue + + +def _clear_bootstrap_continuation_streak(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY) + except Exception: + logger.warning("Failed to clear bootstrap continuation streak", exc_info=True) + + +def _bump_hourly_failure_streak(redis_client: Redis) -> int: + """Инкрементирует счётчик ошибок hourly. Возвращает новое значение.""" + try: + streak = int(redis_client.incr(HOURLY_FAILURE_STREAK_KEY)) + redis_client.expire(HOURLY_FAILURE_STREAK_KEY, HOURLY_FAILURE_STREAK_TTL_SECONDS) + return streak + except Exception: + logger.warning("Failed to update hourly failure streak", exc_info=True) + return 0 + + +def _check_hourly_circuit_breaker(redis_client: Redis) -> tuple[int, bool]: + """Проверяет открыт ли circuit breaker. Возвращает (streak, is_open).""" + try: + raw = redis_client.get(HOURLY_FAILURE_STREAK_KEY) + streak = int(raw) if raw else 0 + except Exception: + return 0, False + is_open = streak >= HOURLY_FAILURE_STREAK_LIMIT + if is_open: + logger.error("Hourly circuit breaker OPEN (%d/%d failures) — skipping", streak, HOURLY_FAILURE_STREAK_LIMIT) + return streak, is_open + + +def _clear_hourly_failure_streak(redis_client: Redis) -> None: + try: + redis_client.delete(HOURLY_FAILURE_STREAK_KEY) + except Exception: + pass + + +def _start_lock_heartbeat( + redis_client: Redis, + key: str, + owner_token: str, + ttl_seconds: int, + stop_on_lost: bool = True, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) + + def _heartbeat() -> None: + consecutive_failures = 0 + while not stop_event.wait(interval_seconds): + refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) + if refreshed is False: + logger.warning("Lost sync_listing lock ownership for %s", owner_token) + if stop_on_lost: + return + consecutive_failures += 1 + continue + if refreshed is None: + consecutive_failures += 1 + if consecutive_failures >= 5: + logger.error("Lock heartbeat failed %d times in a row for %s; giving up", consecutive_failures, owner_token) + return + else: + consecutive_failures = 0 + + thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) + thread.start() + return stop_event, thread + + +def _reset_segments_progress(redis_client: Redis, total: int) -> None: + try: + pipe = redis_client.pipeline() + pipe.delete(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.set(SYNC_SEGMENTS_TOTAL_KEY, str(int(total)), ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.execute() + except Exception: + logger.warning("Failed to reset segments progress", exc_info=True) + + +def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[int, int]: + """Помечает сегмент завершённым. Возвращает (completed_count, total).""" + try: + pipe = redis_client.pipeline() + pipe.sadd(SYNC_SEGMENTS_PROGRESS_KEY, str(int(segment_index))) + pipe.expire(SYNC_SEGMENTS_PROGRESS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.scard(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.get(SYNC_SEGMENTS_TOTAL_KEY) + results = pipe.execute() + completed = int(results[2] or 0) + total = int(results[3] or 0) if results[3] else 0 + return completed, total + except Exception: + logger.warning("Failed to mark segment %d completed", segment_index, exc_info=True) + return 0, 0 + + +def _build_listing_segments(settings: Settings) -> list[dict[str, str | int | None]]: + """Возвращает сегменты листинга с учётом runtime_config. + + При IAAI_LISTING_SEGMENTS=runtime сегменты строятся из filters.brands / filters.include.brands. + Остальные значения IAAI_LISTING_SEGMENTS сохраняют прежнее поведение: auto или JSON. + """ + filtered_urls = settings.listing.filtered_search_urls + if len(filtered_urls) > 1: + return [ + {"make": None, "year_min": None, "year_max": None, "listing_url": url} + for url in filtered_urls + ] + if len(filtered_urls) == 1: + return [] + + raw_segments = settings.listing.listing_segments_json.strip() + if raw_segments.casefold() != "runtime": + return parse_listing_segments(raw_segments) + + parsed_override = parse_listing_segments(raw_segments) + if parsed_override: + return parsed_override + + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + brands = runtime_config.filters.include.brands + if settings.scraping_profile.http_first and settings.listing.fast_segment_year_splits: + segments = build_fast_listing_segments_for_makes(list(brands)) + else: + segments = build_listing_segments_for_makes(list(brands)) + if not segments: + logger.warning( + "IAAI_LISTING_SEGMENTS=runtime, but runtime_config filters.brands is empty; segmented listing disabled" + ) + return segments + + +@shared_task( + name="iaai_scraper.worker.tasks.sync_segment_task", + queue=IAAI_SYNC_QUEUE, + bind=True, + max_retries=2, + default_retry_delay=60, + acks_late=True, +) +def sync_segment_task( + self, + segment_index: int, + segment: dict, + lane: str = "iaai_cars", + only_new: bool | None = None, + is_bootstrap: bool = False, +): + """Обработка одного сегмента листинга. Запускается параллельно несколькими воркерами.""" + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + redis_client = _get_redis() + settings = Settings() + + # Per-segment lock — защита от случайного дубля + seg_lock_key = SYNC_SEGMENT_LOCK_KEY_FMT.format(idx=int(segment_index)) + owner_token = f"{task_id}:{uuid.uuid4().hex}" + lock_ttl = _sync_segment_lock_ttl_seconds() + lock_acquired = _acquire_lock(redis_client, seg_lock_key, owner_token, lock_ttl) + if not lock_acquired: + logger.info("sync_segment_task[%d] skipped: already running", segment_index) + return {"status": "skipped", "segment_index": segment_index, "reason": "duplicate"} + + seg_make = segment.get("make") + seg_year_min = segment.get("year_min") + seg_year_max = segment.get("year_max") + seg_listing_url = segment.get("listing_url") + seg_label = f"{seg_make or 'ALL'}" + if seg_listing_url: + seg_label = "FILTERED_SEARCH" + if seg_year_min is not None or seg_year_max is not None: + seg_label += f" ({seg_year_min}-{seg_year_max})" + + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_started", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(redis_client, seg_lock_key, owner_token, lock_ttl) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=seg_lock_key, + lock_owner=owner_token, + ) + + try: + def _job(): + with IAAIScraper() as scraper: + scraper.set_progress_callback( + lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + **meta, + ) + ) + return scraper.sync_listing( + make=seg_make, + model=None, + lane=lane, + only_new=only_new, + listing_url=str(seg_listing_url) if seg_listing_url else None, + year_min=seg_year_min, + year_max=seg_year_max, + skip_mark_sold=True, + ) + + result = _run_browser_job(_job) + segment_done = bool(result.get("full_scan_completed", False)) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_completed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + status=result.get("status", "success"), + cars_upserted=result.get("cars_upserted", 0), + cars_failed=result.get("cars_failed", 0), + ) + + if is_bootstrap and segment_done: + completed, total = _mark_segment_completed(redis_client, segment_index) + logger.warning( + "Segment %d (%s) bootstrap done: %d/%d completed", + segment_index, seg_label, completed, total, + ) + if total > 0 and completed >= total: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + logger.warning("All %d segments completed; bootstrap full scan done", total) + + return { + "status": result.get("status", "success"), + "segment_index": segment_index, + "segment_label": seg_label, + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), + "full_scan_completed": segment_done, + } + + except SoftTimeLimitExceeded: + logger.warning("sync_segment_task[%d] soft timeout — partial progress saved", segment_index) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_soft_timeout", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + return { + "status": "timed_out", + "segment_index": segment_index, + "segment_label": seg_label, + } + except Exception as exc: + logger.error("sync_segment_task[%d] failed: %s — %s", segment_index, seg_label, exc, exc_info=True) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_failed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + error=str(exc), + ) + try: + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + return { + "status": "failed", + "segment_index": segment_index, + "segment_label": seg_label, + "error": str(exc), + } + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=1) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=1) + _clear_task_progress(redis_client, task_id) + _release_lock_if_owner(redis_client, seg_lock_key, owner_token) + + +@shared_task( + name="iaai_scraper.worker.tasks.sync_vehicle_task", + queue=IAAI_SYNC_QUEUE, + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"): + # Скрапинг и upsert одного автомобиля. + persistence = _get_persistence() + persistence.create_tables() + + try: + def _job(): + with IAAIScraper() as scraper: + return scraper.sync_vehicle(vehicle_url, lane=lane) + + result = _run_browser_job(_job) + logger.info("sync_vehicle_task completed: %s", vehicle_url) + return { + "status": "success", + "vehicle_url": vehicle_url, + "db_action": result.get("db_action"), + "images_upserted": result.get("images_upserted", 0), + } + + except Exception as exc: + logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True) + raise self.retry(exc=exc) + + +@shared_task( + name=MOBILEDE_RUNTIME_SEGMENTS_TASK, + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=1, + default_retry_delay=30, + acks_late=True, +) +def mobilede_sync_runtime_segments_task( + self, + lane: str = "mobile_de_cars", + delay_seconds: float = 0.7, + use_cursor: bool = True, + continuous: bool = True, +): + redis_client = _get_redis() + owner_token = self.request.id or uuid.uuid4().hex + if not redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY, owner_token, nx=True, ex=30 * 60): + logger.info("mobile.de runtime segment rebuild already running") + return {"status": "building"} + try: + cached_segments = _get_cached_mobilede_runtime_segments(redis_client) + if not cached_segments: + settings = Settings() + cached_segments = _build_mobilede_runtime_segments(settings) + redis_client.set(MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY, json.dumps(cached_segments, ensure_ascii=False), ex=24 * 60 * 60) + redis_client.set(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY, str(len(cached_segments))) + redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY) + logger.info("mobile.de runtime segments rebuilt: %s", len(cached_segments)) + segments = _enqueue_mobilede_runtime_segments( + lane=lane, + delay_seconds=delay_seconds, + use_cursor=use_cursor, + continuous=continuous, + ) + if not segments: + logger.info("mobilede_sync_runtime_segments_task: runtime segments not configured, falling back to generic sync") + mobilede_sync_search_task.apply_async( + kwargs={ + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": continuous, + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + return {"status": "fallback", "segments": 0} + logger.info( + "mobilede_sync_runtime_segments_task queued %s runtime segment(s): %s", + len(segments), + ", ".join(str(item.get("label") or item.get("make_id") or "segment") for item in segments), + ) + return { + "status": "queued", + "segments": len(segments), + "labels": [str(item.get("label") or item.get("make_id") or "segment") for item in segments], + } + except Exception as exc: + logger.error("mobilede_sync_runtime_segments_task failed: %s", exc, exc_info=True) + raise self.retry(exc=exc) + finally: + try: + if redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY) == owner_token: + redis_client.delete(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY) + except Exception: + logger.debug("Failed to release mobile.de runtime segment rebuild lock", exc_info=True) + + +@shared_task( + name="mobilede.sync_detail", + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def mobilede_sync_detail_task(self, listing_id: str, lane: str = "mobile_de_cars"): + try: + scraper = MobileDeScraper(persistence=_get_persistence()) + result = scraper.sync_detail(str(listing_id), lane=lane) + logger.info("mobilede_sync_detail_task completed: %s", listing_id) + return {"status": "success", **result} + except Exception as exc: + logger.error("mobilede_sync_detail_task failed: %s — %s", listing_id, exc, exc_info=True) + raise self.retry(exc=exc) + + +@shared_task( + name="mobilede.sync_search", + queue=MOBILEDE_SYNC_QUEUE, + bind=True, + max_retries=2, + default_retry_delay=60, + acks_late=True, +) +def mobilede_sync_search_task( + self, + start_page: int = 1, + max_pages: int = 5, + lane: str = "mobile_de_cars", + only_new: bool | None = None, + search_url: str | None = None, + make_id: str | None = None, + model_id: str | None = None, + price_min: str | None = None, + price_max: str | None = None, + year_min: str | None = None, + year_max: str | None = None, + mileage_min: str | None = None, + mileage_max: str | None = None, + delay_seconds: float = 0.7, + use_cursor: bool = False, + continuous: bool | None = None, + segment: dict | None = None, + segment_index: int | None = None, + runtime_rotation: bool = False, +): + task_id = self.request.id or "unknown" + redis_client = _get_redis() + settings = Settings() + runtime_config = RuntimeConfig.from_file(settings.runtime_config_file) + if only_new is None and runtime_config.sync.only_new is not None: + only_new = runtime_config.sync.only_new + runtime_segments_enabled = False + if segment is None and not make_id and not model_id: + reserved_segment = _reserve_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if reserved_segment is not None: + segment_index, segment = reserved_segment + runtime_segments_enabled = True + else: + if not redis_client.get(MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY): + mobilede_sync_runtime_segments_task.apply_async( + kwargs={ + "lane": lane, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": bool(continuous if continuous is not None else MOBILEDE_CONTINUOUS_SYNC_ENABLED), + }, + queue=MOBILEDE_SYNC_QUEUE, + ) + logger.info("mobile.de runtime segments are not ready; deferring sync task") + raise self.retry(countdown=30) + elif segment is not None: + runtime_segments_enabled = True + + if segment: + search_url = search_url or str(segment.get("search_url") or segment.get("listing_url") or "").strip() or None + make_id = make_id or str(segment.get("make_id") or "").strip() or None + model_id = model_id or str(segment.get("model_id") or "").strip() or None + price_min = price_min or str(segment.get("price_min") or "").strip() or None + price_max = price_max or str(segment.get("price_max") or "").strip() or None + year_min = year_min or str(segment.get("year_min") or "").strip() or None + year_max = year_max or str(segment.get("year_max") or "").strip() or None + mileage_min = mileage_min or str(segment.get("mileage_min") or "").strip() or None + mileage_max = mileage_max or str(segment.get("mileage_max") or "").strip() or None + if segment.get("only_new") is not None: + only_new = bool(segment.get("only_new")) + start_page = int(segment.get("start_page") or start_page or 1) + if segment.get("max_pages") is not None: + max_pages = int(segment.get("max_pages") or max_pages) + if _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + start_page = 1 + max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) + elif make_id or model_id: + resolved_segment = _find_mobilede_runtime_segment( + settings, + search_url=search_url, + make_id=make_id, + model_id=model_id, + ) + if resolved_segment is not None: + segment = resolved_segment + runtime_segments_enabled = True + elif search_url: + resolved_segment = _find_mobilede_runtime_segment( + settings, + search_url=search_url, + make_id=make_id, + model_id=model_id, + ) + if resolved_segment is not None: + segment = resolved_segment + runtime_segments_enabled = True + + if segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + start_page = 1 + max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) + use_cursor = False + + strict_first_pass_mode = _mobilede_is_strict_first_pass_mode(redis_client, segment=segment, only_new=only_new) + cycle_id: str | None = None + if strict_first_pass_mode: + segment, segment_index, cycle_id = _mobilede_reserve_strict_first_pass_segment( + redis_client, + settings, + segment=segment, + segment_index=segment_index, + ) + start_page = 1 + max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) + use_cursor = False + + sort_by: str | None = None + sort_order: str | None = None + if only_new and MOBILEDE_ONLY_NEW_NEWEST_FIRST: + sort_by = "doc" + sort_order = "down" + search_url = _mobilede_apply_newest_sort_to_url(search_url) + + cursor_key = _mobilede_cursor_key(segment) + if strict_first_pass_mode and cycle_id: + cursor_key = _mobilede_cycle_cursor_key(cursor_key, cycle_id) + actual_start_page, actual_end_page = _reserve_mobilede_page_window( + redis_client, + requested_start_page=start_page, + page_window_size=max_pages, + use_cursor=use_cursor, + cursor_key=cursor_key, + ) + if use_cursor and MOBILEDE_SKIP_EMPTY_WINDOW and actual_start_page > 100: + _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) + actual_start_page, actual_end_page = _reserve_mobilede_page_window( + redis_client, + requested_start_page=1, + page_window_size=max_pages, + use_cursor=use_cursor, + cursor_key=cursor_key, + ) + logger.info( + "mobile.de cursor wrapped before empty window: runtime=%s pages=%s-%s", + _mobilede_segment_label(segment), + actual_start_page, + actual_end_page, + ) + _update_task_progress( + redis_client, + task_id=task_id, + stage="mobilede_sync_started", + ttl_seconds=3600, + start_page=actual_start_page, + end_page=actual_end_page, + requested_start_page=start_page, + max_pages=max_pages, + use_cursor=use_cursor, + segment_index=segment_index, + segment_label=(segment or {}).get("label") if segment else None, + ) + try: + if continuous is None: + continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED + segment_label = _mobilede_segment_label(segment) + make_name = _mobilede_segment_make(segment, make_id) + model_name = _mobilede_segment_model(segment, model_id) + logger.info( + "mobile.de sync started: runtime=%s filter=%s pages=%s-%s max_pages=%s use_cursor=%s continuous=%s sort=%s:%s", + segment_label, + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + max_pages, + use_cursor, + continuous, + sort_by, + sort_order, + ) + logger.debug( + "mobilede_sync_search_task started: task_id=%s segment=%s start_page=%s end_page=%s max_pages=%s use_cursor=%s continuous=%s only_new=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s", + task_id, + segment_label, + actual_start_page, + actual_end_page, + max_pages, + use_cursor, + continuous, + only_new, + bool(search_url), + make_id, + model_id, + year_min, + year_max, + price_min, + price_max, + mileage_min, + mileage_max, + ) + + window_pages_collected = 0 + + def _progress(stage: str, meta: dict[str, object]) -> None: + nonlocal window_pages_collected + _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=3600, + start_page=actual_start_page, + end_page=actual_end_page, + use_cursor=use_cursor, + segment_index=segment_index, + segment_label=(segment or {}).get("label") if segment else None, + **meta, + ) + if stage == "search_collection_done": + window_pages_collected = int(meta.get("pages_collected", 0) or 0) + elif stage == "db_upsert_done": + _log_mobilede_progress_threshold( + redis_client, + task_id=task_id, + segment=segment, + delta_pages=window_pages_collected, + delta_cars=int(meta.get("inserted", 0) or 0) + int(meta.get("updated", 0) or 0), + delta_images=int(meta.get("images_upserted", 0) or 0), + start_page=actual_start_page, + end_page=actual_end_page, + ) + + scraper = MobileDeScraper( + client=MobileDeClient.for_worker(delay_seconds=delay_seconds), + persistence=_get_persistence(), + ) + result = scraper.sync_search( + start_page=actual_start_page, + max_pages=max_pages, + lane=lane, + only_new=only_new, + sort_by=sort_by, + sort_order=sort_order, + search_url=search_url, + make_id=make_id, + model_id=model_id, + price_min=price_min, + price_max=price_max, + year_min=year_min, + year_max=year_max, + mileage_min=mileage_min, + mileage_max=mileage_max, + progress_callback=_progress, + ) + inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0) + _mobilede_update_segment_freshness_state( + redis_client, + segment=segment, + only_new=only_new, + inserted=inserted_count, + listings=int(result.get("listing_count", 0) or 0), + ) + listing_count = int(result.get("listing_count", 0) or 0) + if use_cursor and _mobilede_segment_scan_complete(redis_client, segment): + total_segments_raw = redis_client.get(MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY) + total_segments = int(total_segments_raw) if total_segments_raw else None + _mark_mobilede_bootstrap_segment_done(redis_client, segment, total_segments) + if use_cursor and listing_count == 0: + _reset_mobilede_page_cursor(redis_client, cursor_key=cursor_key, next_start_page=1) + logger.debug( + "mobile.de cursor reset after empty window: task_id=%s segment=%s start_page=%s end_page=%s", + task_id, + (segment or {}).get("label") if segment else None, + actual_start_page, + actual_end_page, + ) + _update_task_progress( + redis_client, + task_id=task_id, + stage="sync_done", + ttl_seconds=3600, + cars_upserted=result.get("upsert", {}).get("inserted", 0) + result.get("upsert", {}).get("updated", 0), + listing_count=result.get("listing_count", 0), + start_page=actual_start_page, + end_page=actual_end_page, + use_cursor=use_cursor, + segment_index=segment_index, + segment_label=(segment or {}).get("label") if segment else None, + ) + logger.debug( + "mobilede_sync_search_task completed: task_id=%s segment=%s start_page=%s end_page=%s listings=%s inserted=%s updated=%s", + task_id, + segment_label, + actual_start_page, + actual_end_page, + result.get("listing_count"), + result.get("upsert", {}).get("inserted", 0), + result.get("upsert", {}).get("updated", 0), + ) + logger.info( + "mobile.de sync completed: runtime=%s filter=%s pages=%s-%s listings=%s unique=%s inserted=%s updated=%s images=%s run_id=%s", + segment_label, + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + int(result.get("listing_count", 0) or 0), + int(result.get("unique_listing_count", 0) or 0), + inserted_count, + int(result.get("upsert", {}).get("updated", 0) or 0), + int(result.get("upsert", {}).get("images_upserted", 0) or 0), + result.get("run_id"), + ) + if continuous: + incremental_mode = bool(segment and _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP) + next_start_page = 1 if incremental_mode else (actual_end_page + 1 if listing_count > 0 else 1) + next_max_pages = min(max_pages, MOBILEDE_INCREMENTAL_PAGE_WINDOW) if incremental_mode else max_pages + followup_kwargs = { + "start_page": next_start_page, + "max_pages": next_max_pages, + "lane": lane, + "search_url": search_url, + "make_id": make_id, + "model_id": model_id, + "price_min": price_min, + "price_max": price_max, + "year_min": year_min, + "year_max": year_max, + "mileage_min": mileage_min, + "mileage_max": mileage_max, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "only_new": only_new, + "continuous": True, + "runtime_rotation": runtime_rotation, + } + if runtime_rotation and MOBILEDE_ROTATE_RUNTIME_SEGMENTS: + next_segment_reservation = _reserve_next_mobilede_runtime_segment(redis_client, settings, only_new=only_new) + if next_segment_reservation is not None: + next_segment_index, next_segment = next_segment_reservation + followup_kwargs.update( + { + "start_page": int(next_segment.get("start_page") or 1), + "max_pages": int(next_segment.get("max_pages") or max_pages), + "search_url": str(next_segment.get("search_url") or next_segment.get("listing_url") or "").strip() or None, + "make_id": str(next_segment.get("make_id") or "").strip() or None, + "model_id": str(next_segment.get("model_id") or "").strip() or None, + "price_min": str(next_segment.get("price_min") or "").strip() or None, + "price_max": str(next_segment.get("price_max") or "").strip() or None, + "year_min": str(next_segment.get("year_min") or "").strip() or None, + "year_max": str(next_segment.get("year_max") or "").strip() or None, + "mileage_min": str(next_segment.get("mileage_min") or "").strip() or None, + "mileage_max": str(next_segment.get("mileage_max") or "").strip() or None, + "segment": next_segment, + "segment_index": next_segment_index, + } + ) + if _mobilede_bootstrap_done(redis_client) and MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: + followup_kwargs["start_page"] = 1 + followup_kwargs["max_pages"] = min(int(followup_kwargs["max_pages"]), MOBILEDE_INCREMENTAL_PAGE_WINDOW) + followup_kwargs["use_cursor"] = False + next_start_page = int(followup_kwargs["start_page"]) + logger.info( + "mobile.de runtime rotation queued: current=%s next=%s next_pages=%s-%s", + segment_label, + _mobilede_segment_label(next_segment), + next_start_page, + next_start_page + int(followup_kwargs["max_pages"]) - 1, + ) + elif segment is not None and int(result.get("listing_count", 0) or 0) > 0: + followup_kwargs["segment"] = segment + followup_kwargs["segment_index"] = segment_index + elif segment is not None and runtime_segments_enabled: + followup_kwargs["segment"] = None + followup_kwargs["segment_index"] = None + mobilede_sync_search_task.apply_async( + kwargs=followup_kwargs, + queue=MOBILEDE_SYNC_QUEUE, + countdown=MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS, + ) + logger.debug( + "mobilede_sync_search_task queued follow-up: segment=%s next_start_page=%s delay=%ss use_cursor=%s", + (followup_kwargs.get("segment") or {}).get("label") if isinstance(followup_kwargs.get("segment"), dict) else None, + next_start_page, + MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS, + use_cursor, + ) + logger.info( + "mobile.de sync next window queued: runtime=%s filter=%s next_pages=%s-%s delay=%ss", + segment_label, + _mobilede_filter_source(segment, search_url), + next_start_page, + next_start_page + int(followup_kwargs["max_pages"]) - 1, + MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS, + ) + return _mobilede_task_result_summary( + result=result, + segment=segment, + start_page=actual_start_page, + end_page=actual_end_page, + make_name=make_name, + model_name=model_name, + ) + except Exception as exc: + _update_task_progress( + redis_client, + task_id=task_id, + stage="failed", + ttl_seconds=3600, + error=str(exc), + start_page=actual_start_page, + end_page=actual_end_page, + use_cursor=use_cursor, + ) + is_transient_request_error = _is_mobilede_transient_request_error(exc) + max_retries = int(getattr(self, "max_retries", 0) or 0) + current_retries = int(getattr(self.request, "retries", 0) or 0) + if is_transient_request_error: + logger.warning( + "mobile.de network issue: runtime=%s filter=%s pages=%s-%s retry=%s/%s error=%s", + _mobilede_segment_label(segment), + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + current_retries + 1, + max_retries, + exc, + ) + if current_retries < max_retries: + raise self.retry(exc=exc, countdown=max(60, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS)) + + if continuous is None: + continuous = MOBILEDE_CONTINUOUS_SYNC_ENABLED + if continuous: + followup_kwargs = { + "start_page": actual_start_page, + "max_pages": max_pages, + "lane": lane, + "search_url": search_url, + "make_id": make_id, + "model_id": model_id, + "price_min": price_min, + "price_max": price_max, + "year_min": year_min, + "year_max": year_max, + "mileage_min": mileage_min, + "mileage_max": mileage_max, + "delay_seconds": delay_seconds, + "use_cursor": use_cursor, + "continuous": True, + } + if segment is not None: + followup_kwargs["segment"] = segment + followup_kwargs["segment_index"] = segment_index + mobilede_sync_search_task.apply_async( + kwargs=followup_kwargs, + queue=MOBILEDE_SYNC_QUEUE, + countdown=max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4), + ) + logger.warning( + "mobile.de delayed retry queued after network issue: runtime=%s filter=%s pages=%s-%s delay=%ss", + _mobilede_segment_label(segment), + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + max(300, MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS * 4), + ) + return { + "status": "network_error_deferred", + "runtime": _mobilede_segment_label(segment), + "make": _mobilede_segment_make(segment, make_id), + "model": _mobilede_segment_model(segment, model_id), + "pages": { + "start": actual_start_page, + "end": actual_end_page, + "count": actual_end_page - actual_start_page + 1, + }, + "error": str(exc), + } + logger.error( + "mobile.de sync failed: runtime=%s filter=%s pages=%s-%s error=%s", + _mobilede_segment_label(segment), + _mobilede_filter_source(segment, search_url), + actual_start_page, + actual_end_page, + exc, + exc_info=True, + ) + raise self.retry(exc=exc) + + +@shared_task( + name=SYNC_LISTING_TASK_NAME, + queue=IAAI_SYNC_QUEUE, + bind=True, + max_retries=3, + default_retry_delay=120, + acks_late=True, +) +def sync_listing_task( + self, + make: str | None = None, + model: str | None = None, + lane: str = "iaai_cars", + limit: int | None = None, + only_new: bool | None = None, +): + # Полный цикл: листинг + sync всех найденных машин. + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + owner_token = f"{task_id}:{uuid.uuid4().hex}" + redis_client = _get_redis() + + lock_acquired = False + lock_ttl = _sync_listing_lock_ttl_seconds() + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + force_bootstrap_full_scan = False + + def _enqueue_bootstrap_followup( + reason: str, + delay_seconds: int = 5, + *, + count_as_failure: bool = False, + ) -> None: + flag_ttl = max(lock_ttl, delay_seconds + 300) + if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl): + logger.info( + "Bootstrap follow-up already pending; skip enqueue (reason=%s)", + reason, + ) + return + if count_as_failure: + _, should_enqueue = _bump_bootstrap_failure_streak(redis_client, reason=reason) + if not should_enqueue: + _clear_followup_pending(redis_client) + return + else: + _clear_bootstrap_failure_streak(redis_client) + + continuation_streak, should_continue = _bump_bootstrap_continuation_streak(redis_client) + if not should_continue: + _clear_followup_pending(redis_client) + # Переключаемся на часовой beat-режим и останавливаем + # немедленные bootstrap continuation, чтобы не зациклиться. + if not always_full_scan: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + logger.error( + "Bootstrap continuation stopped after %d immediate runs; switching to hourly schedule", + continuation_streak, + ) + else: + logger.error( + "Bootstrap continuation stopped after %d immediate runs (always_full_scan=true)", + continuation_streak, + ) + return + + try: + followup_expires = max(int(lock_ttl), int(delay_seconds) + 300) + self.app.send_task( + SYNC_LISTING_TASK_NAME, + kwargs={ + "make": make, + "model": model, + "lane": lane, + "limit": limit, + "only_new": only_new, + }, + queue=IAAI_SYNC_QUEUE, + countdown=max(0, int(delay_seconds)), + expires=followup_expires, + ) + logger.info( + "Bootstrap follow-up sync queued in %ss (reason=%s)", + delay_seconds, + reason, + ) + except Exception: + _clear_followup_pending(redis_client) + logger.warning("Failed to enqueue bootstrap follow-up sync", exc_info=True) + + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id) + if orphan_cleared: + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + logger.info("sync_listing_task skipped: another sync is already running") + return { + "status": "skipped", + "reason": "sync_already_running", + "task_id": task_id, + } + + try: + settings = Settings() + + # Любой реально стартовавший sync_listing снимает pending-флаг followup, + # чтобы watchdog/continuation могли корректно планировать следующий run + # только при новой проблеме, а не копить дубликаты в очереди. + _clear_followup_pending(redis_client) + + # Start heartbeat + stall watchdog immediately after lock acquisition + # so ALL code paths (hourly, bootstrap, segmented) are protected. + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, + SYNC_LISTING_LOCK_KEY, + owner_token, + lock_ttl, + stop_on_lost=False, + ) + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + _update_task_progress( + redis_client, + task_id=task_id, + stage="sync_listing_started", + ttl_seconds=progress_ttl, + ) + + full_scan_done_before_run = _is_full_scan_done(redis_client) + always_full_scan = bool(settings.discovery.always_full_scan) + explicit_filtered_run = bool( + make is not None + or model is not None + or (limit is not None and int(limit or 0) > 0) + or only_new is True + ) + force_bootstrap_full_scan = (always_full_scan or (not full_scan_done_before_run)) and not explicit_filtered_run + if explicit_filtered_run and not full_scan_done_before_run: + logger.info( + "Explicit sync_listing request detected; honoring make/model/limit/only_new before bootstrap full scan is complete", + ) + effective_limit = None if force_bootstrap_full_scan else limit + effective_only_new = False if force_bootstrap_full_scan else only_new + hourly_mode = settings.discovery.hourly_mode.strip().lower() + discovery_mode = settings.discovery.mode.strip().lower() + if always_full_scan: + # Для режима "полный прогон каждый запуск" приоритет — устойчивый resume, + # поэтому принудительно уходим в listing/segmented path вместо sitemap-mainline. + discovery_mode = "listing" + prefer_sitemap_mainline = ( + not force_bootstrap_full_scan + and make is None + and model is None + and effective_limit is None + and not effective_only_new + and not always_full_scan + ) + # Определяем сегменты из конфига/env или из runtime_config при IAAI_LISTING_SEGMENTS=runtime. + filtered_listing_urls = settings.listing.filtered_search_urls + filtered_listing_url = filtered_listing_urls[0] if filtered_listing_urls else None + segments = _build_listing_segments(settings) + + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=SYNC_LISTING_LOCK_KEY, + lock_owner=owner_token, + db_idle_restart_seconds=(DB_IDLE_RESTART_SECONDS if segments else None), + ) + use_hourly_sitemap_sync = ( + (not always_full_scan) + and full_scan_done_before_run + and prefer_sitemap_mainline + and not segments + ) + + # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. + # Используется только во время bootstrap для пропуска уже обработанных сегментов. + # Никаких page-level resume — внутри сегмента всегда стартуем с page 1. + last_completed_segment: int | None = None + if force_bootstrap_full_scan and (always_full_scan or not full_scan_done_before_run): + last_completed_segment = _load_last_completed_segment(redis_client) + else: + # После завершения bootstrap чекпоинт не нужен никогда. + _clear_sync_checkpoint(redis_client) + + if force_bootstrap_full_scan: + logger.info( + "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run", + ) + if always_full_scan: + logger.info( + "Always full scan mode enabled (IAAI_ALWAYS_FULL_SCAN=true): using listing resume path", + ) + + logger.info( + "sync_listing options: only_new=%s, limit=%s", + effective_only_new, + effective_limit, + ) + + if use_hourly_sitemap_sync: + # Circuit breaker: если N подряд hourly-запусков фейлили, пропускаем. + _hourly_streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if _cb_open: + return { + "status": "circuit_breaker_open", + "task_id": task_id, + "hourly_failure_streak": _hourly_streak, + } + _progress_cb = lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, + ) + try: + if hourly_mode == "diff": + logger.info("Hourly mode: running sitemap diff sync instead of full listing traversal") + result = _hourly_sitemap_diff_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + elif hourly_mode == "full_refresh": + logger.info("Hourly mode: running sitemap full refresh of all active vehicles") + result = _hourly_sitemap_full_refresh_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + else: + logger.info("Hourly mode: running sitemap rolling refresh of active vehicles") + result = _hourly_sitemap_rolling_refresh_sync( + redis_client=redis_client, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + except SitemapDiscoveryError as exc: + logger.warning( + "Sitemap discovery failed (%s); falling back to listing traversal for hourly sync", + exc, + ) + use_hourly_sitemap_sync = False # fall through to listing traversal below + prefer_sitemap_mainline = False # allow segmented fallback + discovery_mode = "listing" # override so use_segmented check passes + + if use_hourly_sitemap_sync: + try: + redis_client.set(SITEMAP_HOURLY_LAST_COUNT_KEY, str(result.get("discovered_urls", 0))) + except Exception: + logger.warning("Failed to persist hourly sitemap count", exc_info=True) + + # Anti-bot guard: при массовом protection/failed не считаем запуск успешным, + # открываем hourly circuit breaker и уходим в controlled retry по beat. + hourly_failures = len(result.get("failures") or []) + hourly_discovered = int(result.get("discovered_urls") or 0) + hourly_failed = int(result.get("cars_failed") or 0) + hourly_protection = int(result.get("protection_events") or 0) + if hourly_discovered > 0: + fail_ratio = hourly_failed / max(1, hourly_discovered) + protection_ratio = hourly_protection / max(1, hourly_discovered) + anti_bot_suspected = ( + (hourly_protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + _streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered: discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + hourly_discovered, + hourly_failed, + hourly_protection, + fail_ratio, + protection_ratio, + _streak, + ) + return { + "status": "anti_bot_detected", + "task_id": task_id, + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": hourly_discovered, + "cars_failed": hourly_failed, + "protection_events": hourly_protection, + "hourly_failure_streak": _streak, + } + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": result.get("discovered_urls", 0), + "new_urls": result.get("new_urls", 0), + "sold_marked": result.get("sold_marked", 0), + } + logger.info( + "sync_listing_task hourly diff completed: status=%s, new=%d, sold=%d, skipped=%d", + summary["status"], + summary["new_urls"], + summary["sold_marked"], + summary["skipped_existing"], + ) + # Hourly успешно — сбрасываем circuit breaker streak. + _clear_hourly_failure_streak(redis_client) + return summary + + _clear_followup_pending(redis_client) + + self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) + + use_segmented = ( + bool(segments) + and make is None + and model is None + and not use_hourly_sitemap_sync + ) + + if prefer_sitemap_mainline: + if discovery_mode != "sitemap": + logger.warning( + "Unfiltered full scan forcing sitemap discovery despite IAAI_DISCOVERY_MODE=%s", + discovery_mode or "unset", + ) + if segments: + logger.info("Ignoring configured listing segments for unfiltered sitemap full scan") + + # --- Параллельный диспатч сегментов: dispatch & exit --- + if use_segmented and settings.celery.parallel_segments: + # Сегменты уже завершённые (для bootstrap resume) пропускаем по Redis SET. + already_completed: set[int] = set() + if force_bootstrap_full_scan: + try: + raw = redis_client.smembers(SYNC_SEGMENTS_PROGRESS_KEY) or set() + already_completed = {int(x) for x in raw if str(x).strip().lstrip("-").isdigit()} + except Exception: + already_completed = set() + + pending = [ + (idx, seg) for idx, seg in enumerate(segments) + if idx not in already_completed + ] + + if not pending: + # Всё уже сделано — фиксируем bootstrap done. + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + logger.warning("Parallel segments: nothing to dispatch (all completed)") + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": 0, + "segments_already_completed": len(already_completed), + } + + # При первом запуске bootstrap фиксируем total, чтобы знать когда остановиться. + if force_bootstrap_full_scan and not already_completed: + _reset_segments_progress(redis_client, len(segments)) + + dispatched = 0 + for idx, seg in pending: + try: + self.app.send_task( + "iaai_scraper.worker.tasks.sync_segment_task", + kwargs={ + "segment_index": idx, + "segment": seg, + "lane": lane, + "only_new": effective_only_new, + "is_bootstrap": force_bootstrap_full_scan, + }, + queue=IAAI_SYNC_QUEUE, + ) + dispatched += 1 + except Exception: + logger.warning("Failed to dispatch segment %d", idx, exc_info=True) + + logger.warning( + "Parallel segments dispatched: %d/%d (already_completed=%d, bootstrap=%s)", + dispatched, len(segments), len(already_completed), force_bootstrap_full_scan, + ) + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": dispatched, + "segments_already_completed": len(already_completed), + } + # --- конец параллельной ветки --- + + resume_from_segment = 0 + if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None: + resume_from_segment = max(0, last_completed_segment + 1) + if resume_from_segment >= len(segments): + # Все сегменты уже пройдены — чекпоинт устарел, начинаем заново. + logger.info( + "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0", + last_completed_segment, len(segments), + ) + resume_from_segment = 0 + _clear_sync_checkpoint(redis_client) + elif resume_from_segment > 0: + logger.warning( + "Resuming segmented bootstrap from segment=%d (last completed=%d)", + resume_from_segment, last_completed_segment, + ) + + def _progress_cb_main(stage, meta): + _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, + ) + + def _job(): + with IAAIScraper() as scraper: + scraper.set_progress_callback(_progress_cb_main) + if use_segmented: + return scraper.sync_listing_segmented( + segments=segments, + lane=lane, + only_new=effective_only_new, + start_segment=resume_from_segment, + start_page=1, + progress_callback=( + (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx)) + if force_bootstrap_full_scan and (always_full_scan or not full_scan_done_before_run) else None + ), + ) + return scraper.sync_listing( + make=make, + model=model, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + listing_url=filtered_listing_url, + ) + + result = _run_browser_job(_job) + + if force_bootstrap_full_scan: + bootstrap_completed = bool(result.get("full_scan_completed")) + if bootstrap_completed: + _set_full_scan_done(redis_client, False if always_full_scan else True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + _clear_bootstrap_continuation_streak(redis_client) + if always_full_scan: + logger.info("Full scan completed; keeping bootstrap mode for next run (always full scan enabled)") + else: + logger.info("Bootstrap full scan completed; hourly schedule continues") + else: + _set_full_scan_done(redis_client, False) + listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {} + anti_bot_detected = bool(result.get("anti_bot_detected")) + had_progress = any( + int(result.get(key) or 0) > 0 + for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered") + ) or int(listing_payload.get("vehicles_collected") or 0) > 0 + count_as_failure = anti_bot_detected or (str(result.get("status") or "") == "failed" and not had_progress) + followup_delay = 5 + followup_reason = "bootstrap_not_completed" + if anti_bot_detected: + followup_delay = 180 + followup_reason = "bootstrap_anti_bot_detected" + logger.warning( + "Bootstrap anti-bot guard: protection_events=%s fail_ratio=%s protection_ratio=%s; scheduling delayed continuation", + result.get("protection_events"), + result.get("fail_ratio"), + result.get("protection_ratio"), + ) + else: + logger.info("Bootstrap full scan not complete yet; queuing immediate continuation") + _enqueue_bootstrap_followup( + followup_reason, + delay_seconds=followup_delay, + count_as_failure=count_as_failure, + ) + else: + _clear_sync_checkpoint(redis_client) + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "protection_events": result.get("protection_events", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + } + if not force_bootstrap_full_scan: + discovered = int(result.get("total_discovered") or result.get("total") or 0) + failed = int(summary["cars_failed"] or 0) + protection = int(summary["protection_events"] or 0) + if discovered > 0: + fail_ratio = failed / max(1, discovered) + protection_ratio = protection / max(1, discovered) + anti_bot_suspected = ( + (protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered (listing fallback): discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + discovered, + failed, + protection, + fail_ratio, + protection_ratio, + streak, + ) + summary["status"] = "anti_bot_detected" + summary["hourly_failure_streak"] = streak + return summary + logger.info( + "sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d", + summary["status"], + summary["cars_upserted"], + summary["cars_failed"], + summary["failures_count"], + ) + return summary + + except SoftTimeLimitExceeded: + logger.warning( + "sync_listing_task soft timeout exceeded — partial progress already saved to DB" + ) + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=False) + else: + # Hourly: ставим продолжение, но только если circuit breaker не открыт. + _bump_hourly_failure_streak(redis_client) + _streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if not _cb_open: + followup_ttl = max(600, int(lock_ttl)) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + try: + self.app.send_task( + SYNC_LISTING_TASK_NAME, + kwargs={ + "make": make, + "model": model, + "lane": lane, + "limit": limit, + "only_new": only_new, + }, + queue=IAAI_SYNC_QUEUE, + countdown=10, + expires=followup_ttl, + ) + logger.info("Queued immediate continuation after soft timeout") + except Exception: + _clear_followup_pending(redis_client) + logger.warning("Failed to queue continuation after soft timeout", exc_info=True) + else: + logger.info("Continuation after soft timeout already pending; skip duplicate enqueue") + else: + logger.warning("Skipping continuation: hourly circuit breaker open (%d failures)", _streak) + return { + "status": "timed_out", + "task_id": task_id, + "reason": "soft_time_limit_exceeded", + "note": "partial progress saved to DB; continuation queued", + } + + except Exception as exc: + logger.error("sync_listing_task failed: %s", exc, exc_info=True) + # Hourly circuit breaker: фиксируем ошибку. + if not force_bootstrap_full_scan: + _bump_hourly_failure_streak(redis_client) + try: + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + raise self.retry(exc=exc, countdown=5) + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + logger.error("sync_listing_task max retries exceeded, giving up") + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, False) + _enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True) + # Non-bootstrap: НЕ ставим continuation — beat поставит новую задачу + # через beat_sync_interval_minutes. Бесконечный retry при ошибках + # приводит к молотилке запросов и бану. + return { + "status": "failed", + "task_id": task_id, + "error": str(exc), + } + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=5) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) + if lock_acquired: + _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5078e14 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,45 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "mobile-de-scraper" +version = "0.1.0" +description = "mobile.de scraper service with FastAPI, Celery, Playwright and PostgreSQL" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "alembic>=1.14.0", + "celery>=5.4.0", + "fastapi>=0.115.0", + "playwright>=1.53.0", + "psycopg2-binary>=2.9.9", + "pydantic>=2.8.2", + "python-dotenv>=1.0.1", + "redis>=5.2.0", + "requests>=2.32.0", + "sqlalchemy>=2.0.32", + "uvicorn>=0.34.0", + "urllib3>=2.0.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.3.0", + "pytest-cov>=5.0.0", +] + +[project.scripts] +mobilede = "iaai_scraper.cli:main" +iaai = "iaai_scraper.cli:main" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["iaai_scraper*"] + +[tool.pytest.ini_options] +addopts = "-q --disable-warnings" +python_files = ["tests/test_*.py"] +log_cli = false \ No newline at end of file diff --git a/runtime_config.json b/runtime_config.json new file mode 100644 index 0000000..4fa635c --- /dev/null +++ b/runtime_config.json @@ -0,0 +1,114 @@ +{ + "sync": { + "name": null, + "ids_initial_size": null, + "ids_next_size": null, + "ids_max_pages": null, + "condition_check_enabled": false, + "lane": "iaai_cars", + "only_new": true, + "limit": null + }, + "filters": { + "price": { + "min": null, + "max": null + }, + "mileage": { + "min": null, + "max": null + }, + "flags": { + "damaged_only": null, + "run_and_drive": null + }, + "brands": [ + "Acura", + "Alfa Romeo", + "Alpina", + "Aston Martin", + "Audi", + "BMW", + "BMW ALPINA", + "Bentley", + "Bugatti", + "Cadillac", + "Caterham", + "Chevrolet", + "Chrysler", + "Cupra", + "Daimler", + "Dodge", + "Ferrari", + "Ford", + "Genesis", + "Honda", + "Hummer", + "Hyundai", + "Infiniti", + "Isuzu", + "Jaguar", + "Jeep", + "Kia", + "Lamborghini", + "Land Rover", + "Lexus", + "Lincoln", + "Lotus", + "Lucid", + "Maserati", + "Maybach", + "Mazda", + "McLaren", + "Mercedes-Benz", + "Mercury", + "Mini", + "Mitsubishi", + "Nissan", + "Oldsmobile", + "Pagani", + "Plymouth", + "Pontiac", + "Porsche", + "Ram", + "Ravon", + "Rivian", + "Rolls-Royce", + "Rolls Royce", + "Rover", + "Saturn", + "Skoda", + "Smart", + "Subaru", + "Suzuki", + "Tesla", + "Toyota", + "Volkswagen", + "Volvo", + "KTM AG", + "Koenigsegg", + "Rimac" + ], + "models": [], + "years": [], + "body_types": [], + "colors": [], + "drives": [], + "gearboxes": [], + "locations": [], + "exclude_brands": [], + "exclude_models": [], + "exclude_years": [], + "exclude_body_types": [] + }, + "mobilede": { + "segments": [ + { + "label": "mobile.de ready filter URL", + "search_url": "https://www.mobile.de/ru/транспортные-средства/поиск.html?isSearchRequest=true&s=Car&vc=Car&ref=dsp&ms=3500&ms=1900&ms=5600&ms=11900&ms=24100&ms=25100&ms=20100&ms=23600&pageNumber=1", + "start_page": 1, + "max_pages": 100 + } + ] + } +} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b33f680 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import logging + + +def pytest_configure() -> None: + logging.disable(logging.CRITICAL) diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..7720590 --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from sqlalchemy import select + +from iaai_scraper.core.config import Settings +from iaai_scraper.storage.db import PersistenceService +from iaai_scraper.storage.models import Car, Image, SyncRun +from iaai_scraper.storage.schemas import CarRecord, ImageRecord + + +class TestPersistenceServiceIntegration(unittest.TestCase): + def setUp(self) -> None: + self.tmp_dir = tempfile.TemporaryDirectory() + db_path = Path(self.tmp_dir.name) / "test.sqlite" + + self.settings = Settings() + self.settings.database.url = f"sqlite:///{db_path.as_posix()}" + self.settings.database.echo = False + + self.persistence = PersistenceService(self.settings) + self.persistence.create_tables() + + def tearDown(self) -> None: + self.persistence.engine.dispose() + self.tmp_dir.cleanup() + + @staticmethod + def _record(origin_id: str, *, price: int = 1000) -> CarRecord: + return CarRecord( + parser_id=f"iaai:{origin_id}", + brand="Toyota", + model="Camry", + year=2014, + price=price, + origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + images=[ + ImageRecord( + fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", + preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300", + order_index=0, + ) + ], + ) + + def test_insert_update_and_skip_flow(self) -> None: + first = self._record("777", price=1000) + inserted = self.persistence.upsert_car(first) + self.assertEqual(inserted["action"], "inserted") + self.assertEqual(inserted["images_upserted"], 1) + + same = self._record("777", price=1000) + updated_same = self.persistence.upsert_car(same) + self.assertEqual(updated_same["action"], "updated") + self.assertEqual(updated_same["images_upserted"], 1) + + changed = self._record("777", price=1500) + updated = self.persistence.upsert_car(changed) + self.assertEqual(updated["action"], "updated") + self.assertEqual(updated["images_upserted"], 1) + + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].price, 1500) + self.assertEqual(len(images), 1) + + def test_update_replaces_old_images(self) -> None: + first = self._record("888") + self.persistence.upsert_car(first) + + second = self._record("888") + second.images = [ + ImageRecord( + fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633", + preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + order_index=0, + ) + ] + self.persistence.upsert_car(second) + + with self.persistence.session_scope() as session: + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(images), 1) + self.assertIn("imageKeys=2", images[0].fullres_image) + + def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: + first_run_id = self.persistence.start_sync_run("lane-a") + second_run_id = self.persistence.start_sync_run("lane-b") + + self.assertNotEqual(first_run_id, second_run_id) + + with self.persistence.session_scope() as session: + first = session.get(SyncRun, first_run_id) + second = session.get(SyncRun, second_run_id) + + self.assertEqual(first.status, "failed") + self.assertIsNotNone(first.finished_at) + self.assertEqual(second.status, "running") + + def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: + first = self._record("OLD-ID") + first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + self.persistence.upsert_car(first) + + second = self._record("NEW-ID") + second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + result = self.persistence.upsert_car(second) + + self.assertEqual(result["action"], "updated") + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].origin_id, "NEW-ID") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fast_client.py b/tests/test_fast_client.py new file mode 100644 index 0000000..2f38d28 --- /dev/null +++ b/tests/test_fast_client.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import json + +from iaai_scraper.browser.fast_client import ( + DETAIL_MARKER, + LISTING_MARKER, + build_resizer_images_from_keys, + is_challenge_response, + parse_listing_page, + parse_product_details_vm, +) +from iaai_scraper.parsing.fast_mapper import FastCarMapper + + +def test_parse_listing_page_extracts_hidden_payloads() -> None: + gbp = {"CurrentPage": 1, "PageSize": 100} + vehicles = [ + { + "Id": "45078011~US", + "Tenant": "US", + "InventoryStatus": "RS", + "Currency": "USD", + "PreBidIndicator": True, + } + ] + html = ( + f'' + f'' + '' + '' + '' + ) + + parsed = parse_listing_page(html) + + assert parsed.result_count == 1 + assert parsed.page_size == 100 + assert parsed.current_page == 1 + assert parsed.vehicles[0].inventory_id == "45078011~US" + assert parsed.vehicles[0].inventory_status == "RS" + + +def test_parse_product_details_vm_and_map_record() -> None: + payload = { + "inventoryView": { + "attributes": { + "Id": "45078011~US", + "Year": "2002", + "Make": "ACURA", + "Model": "RSX", + "Series": "BASE", + "Currency": "USD", + "ODOValue": "219187", + "ExteriorColor": "GRAY", + "DriveLineTypeDesc": "FWD", + "Transmission": "Automatic", + "BodyStyleName": "COUPE", + "EngineSize": "2.0L I-4", + "VehicleGrade": "50", + "PrimaryDamageDesc": "NORMAL WEAR & TEAR", + }, + "imageDimensions": { + "keys": { + "$values": [ + {"k": "45078011~US~I1", "w": 1600, "h": 1200, "i": 0}, + ] + } + }, + }, + "auctionInformation": { + "prebidInformation": {"decimalHighBidAmount": "600", "highBidAmount": "$600"}, + "biddingInformation": {"buyNowPrice": "$0"}, + }, + } + html = f'' + + parsed = parse_product_details_vm(html) + record = FastCarMapper().map_payload_to_record( + detail_payload=parsed, + vehicle_url="https://www.iaai.com/VehicleDetail/45078011~US", + ) + + assert record.origin_id == "iaai:45078011~US" + assert record.brand == "ACURA" + assert record.model == "RSX BASE" + assert record.year == 2002 + assert record.price == 600 + assert record.drive == "FWD" + assert record.gearbox == "AT" + assert record.body_type == "COUPE" + assert record.engine_volume == 2000 + assert record.is_damaged is False + assert len(record.images) == 1 + + +def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None: + keys = [ + {"k": "abc~1", "w": 2000, "h": 1500, "i": 2}, + {"k": "abc~1", "w": 2000, "h": 1500, "i": 2}, + {"k": "abc~2", "w": 1800, "h": 1200, "i": 1}, + ] + images = build_resizer_images_from_keys(keys) + + assert len(images) == 2 + assert images[0]["order_index"] == 1 + assert "imageKeys=abc~2" in str(images[0]["fullres_image"]) + + +def test_is_challenge_response_marker_rules() -> None: + assert is_challenge_response(status_code=403, body_text="", expected_marker=None) is True + assert is_challenge_response(status_code=200, body_text="blocked", expected_marker=LISTING_MARKER) is True + assert is_challenge_response( + status_code=200, + body_text=f'{DETAIL_MARKER}', + expected_marker=DETAIL_MARKER, + ) is False diff --git a/tests/test_fast_sync.py b/tests/test_fast_sync.py new file mode 100644 index 0000000..c3e92ef --- /dev/null +++ b/tests/test_fast_sync.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from iaai_scraper.browser.fast_client import FastListingVehicle +from iaai_scraper.core.config import Settings +from iaai_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig +from iaai_scraper.fast_sync import FastSyncEngine +from iaai_scraper.parsing.fast_mapper import FastCarMapper + + +def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle: + return FastListingVehicle( + inventory_id=inventory_id, + tenant="US", + auction_id="1_1", + auction_date="2026-04-08T08:30:00+00:00", + inventory_status=status, + currency="USD", + timed_auction_closed=False, + timed_auction_indicator=False, + prebid_indicator=True, + buynow_indicator=False, + ) + + +def _detail_payload(inventory_id: str, *, make: str = "ACURA", model: str = "RSX", bid: int = 500) -> dict: + return { + "inventoryView": { + "attributes": { + "Id": inventory_id, + "Year": "2002", + "Make": make, + "Model": model, + "Series": "BASE", + "Currency": "USD", + "ODOValue": "219187", + "ExteriorColor": "GRAY", + "DriveLineTypeDesc": "FWD", + "Transmission": "Automatic", + "BodyStyleName": "COUPE", + "EngineSize": "2.0L I-4", + "VehicleGrade": "50", + "PrimaryDamageDesc": "NORMAL WEAR & TEAR", + }, + "imageDimensions": { + "keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]} + }, + }, + "auctionInformation": { + "prebidInformation": {"decimalHighBidAmount": str(bid), "highBidAmount": f"${bid}"}, + "biddingInformation": {"buyNowPrice": "$0"}, + }, + } + + +class FakeFastClient: + def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None: + self.listing = listing + self.details = details + self.detail_calls = 0 + self.persist_calls = 0 + + def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None): # noqa: ANN001, ANN202 + del listing_start_url, make, max_pages + return iter(self.listing) + + def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict: + self.detail_calls += 1 + return self.details[inventory_id] + + def persist_session_state(self) -> None: + self.persist_calls += 1 + + +@dataclass +class FakePersistence: + inserted: int = 0 + images: int = 0 + + def get_existing_urls_and_ids(self, origin_urls, origin_ids): # noqa: ANN001, ANN202 + del origin_urls, origin_ids + return set(), set() + + def upsert_cars_batch(self, records): # noqa: ANN001, ANN202 + self.inserted += len(records) + self.images += sum(len(record.images) for record in records) + return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)} + + def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="iaai"): # noqa: ANN001, ANN202 + del active_origin_urls, lane + return 0 + + +def test_fast_sync_engine_collects_details_and_bulk_upserts() -> None: + listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")] + details = { + "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500), + "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900), + } + client = FakeFastClient(listing, details) + persistence = FakePersistence() + engine = FastSyncEngine( + client=client, # type: ignore[arg-type] + mapper=FastCarMapper(), + persistence=persistence, # type: ignore[arg-type] + batch_size=10, + fetch_concurrency=2, + ) + + result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False) + + assert result["status"] == "success" + assert result["cars_upserted"] == 2 + assert result["images_upserted"] == 2 + assert result["listing"]["mode"] == "fast_hidden_payload" + assert client.detail_calls == 2 + assert client.persist_calls == 1 + + +def test_fast_sync_engine_applies_runtime_filters_before_db() -> None: + listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")] + details = { + "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500), + "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900), + } + runtime = RuntimeConfig(filters=RuntimeFiltersConfig.from_dict({"brands": ["BMW"]})) + persistence = FakePersistence() + engine = FastSyncEngine( + client=FakeFastClient(listing, details), # type: ignore[arg-type] + mapper=FastCarMapper(), + persistence=persistence, # type: ignore[arg-type] + batch_size=10, + fetch_concurrency=2, + ) + + result = engine.sync_listing(runtime_config=runtime, only_new=False) + + assert result["cars_upserted"] == 1 + assert result["cars_filtered"] == 1 + assert persistence.inserted == 1 diff --git a/tests/test_listing.py b/tests/test_listing.py new file mode 100644 index 0000000..7d8e991 --- /dev/null +++ b/tests/test_listing.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.browser.pace import HumanPacer +from iaai_scraper.core.config import Settings +from iaai_scraper.browser.listing import ListingCollector + + +class _FakePage: + def __init__(self, counts: dict[str, int]) -> None: + self._counts = counts + self._evaluate_result = False + self._evaluate_values: list[object] = [] + + class _Locator: + def __init__(self, count_value: int) -> None: + self._count_value = count_value + + def count(self) -> int: + return self._count_value + + def locator(self, selector: str) -> "_FakePage._Locator": + return _FakePage._Locator(self._counts.get(selector, 0)) + + def evaluate(self, _script: str): + if self._evaluate_values: + return self._evaluate_values.pop(0) + return self._evaluate_result + + +class TestListingUnit(unittest.TestCase): + def test_pagination_detection_and_page_number(self) -> None: + # Есть кнопка Next → True. + self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1}))) + # Нет селекторов → False. + self.assertFalse(ListingCollector._has_next_page(_FakePage({}))) + # Числовая пагинация через JS → True только если evaluate явно нашёл next. + page = _FakePage({}) + page._evaluate_result = True + self.assertTrue(ListingCollector._has_next_page(page)) + # Номер текущей страницы. + page2 = _FakePage({}) + page2._evaluate_values = [5] + self.assertEqual(ListingCollector._get_current_page_number(page2), 5) + + def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None: + collector = ListingCollector(Settings(), HumanPacer(Settings())) + html = """ +
+

Car 1

+ +
+ """ + + links = collector._extract_vehicle_links_from_html(html) + + self.assertEqual( + links, + [ + ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"), + ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"), + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py new file mode 100644 index 0000000..621a5fc --- /dev/null +++ b/tests/test_mappers.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.parsing.mapper import CarMapper + + +class TestCarMapper(unittest.TestCase): + def setUp(self) -> None: + self.mapper = CarMapper() + + def test_deduplicates_images_by_image_key(self) -> None: + urls = [ + "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", + "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + ] + + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(len(record.images), 2) + self.assertIn("width=845", record.images[0].fullres_image) + self.assertIn("height=633", record.images[0].fullres_image) + + def test_no_damage_marker_is_not_damaged(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_summary={"make": "Ford", "model": "Focus"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {}, + "damage": {"primary": "normal wear"}, + "auction": {}, + "images": {}, + }, + ) + + self.assertFalse(record.is_damaged) + + def test_unknown_empty_values_and_normalization(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/999~US", + vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + self.assertEqual(record.brand, "UNKNOWN") + self.assertEqual(record.model, "UNKNOWN") + self.assertEqual(record.drive, "NA") + self.assertEqual(record.gearbox, "NA") + + # Нормализация регистра и пробелов. + record2 = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/888~US", + vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + self.assertEqual(record2.drive, "FWD") + self.assertEqual(record2.gearbox, "AT") + + def test_price_and_currency_parsing(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/777~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "USD 4,500 - 5,200"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + self.assertEqual(record.price, 5200) + + record2 = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/778~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "€4.500,00"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + self.assertEqual(record2.currency, "EUR") + self.assertEqual(record2.price, 4500) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..72a2b6c --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.parsing.parser import VehicleParser + + +class TestVehicleParserUnit(unittest.TestCase): + def setUp(self) -> None: + self.parser = VehicleParser() + + def test_parse_dom_pairs_and_title(self) -> None: + dom_text = """ + Stock #: + 45089484 + Primary Damage: + Front End + Odometer: + 50,123 mi (Actual) + """ + result = self.parser._parse_dom_key_value_pairs(dom_text) + self.assertEqual(result.get("lot_number"), "45089484") + self.assertEqual(result.get("primary_damage"), "Front End") + self.assertEqual(result.get("odometer"), "50,123 mi (Actual)") + + parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") + self.assertEqual(parsed["year"], "2014") + self.assertEqual(parsed["make"], "TOYOTA") + self.assertEqual(parsed["model"], "CAMRY") + + def test_extract_image_urls_filters_and_deduplicates(self) -> None: + vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" + payloads = [{"imageUrls": [ + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", + ]}] + urls = self.parser._extract_image_urls(payloads, "", vehicle_url) + self.assertEqual(len(urls), 1) + self.assertIn("45089484", urls[0]) + + def test_dom_hints_and_access_notes_detect_antibot(self) -> None: + hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") + self.assertTrue(hints["has_captcha_text"]) + self.assertTrue(hints["has_antibot_text"]) + + summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."} + notes = self.parser._build_access_notes(summary, []) + self.assertTrue(notes["possible_captcha"]) + self.assertTrue(notes["possible_antibot"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_resilience.py b/tests/test_resilience.py new file mode 100644 index 0000000..8b2bf81 --- /dev/null +++ b/tests/test_resilience.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from iaai_scraper.scraper import IAAIScraper +from iaai_scraper.core.config import Settings +from iaai_scraper.worker import tasks + + +class TestResilience(unittest.TestCase): + def _make_scraper(self) -> IAAIScraper: + s = Settings() + s.log_level = "CRITICAL" + s.database.url = "sqlite://" + return IAAIScraper(s) + + def test_update_task_progress_updates_global_marker(self) -> None: + redis_client = MagicMock() + pipe = MagicMock() + redis_client.pipeline.return_value = pipe + + tasks._update_task_progress( + redis_client, + task_id="task-abc", + stage="batch_upserted", + ttl_seconds=180, + cars_upserted=10, + ) + + redis_client.pipeline.assert_called_once() + self.assertEqual(pipe.set.call_count, 2) + first_call = pipe.set.call_args_list[0] + second_call = pipe.set.call_args_list[1] + + self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc")) + self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY) + pipe.execute.assert_called_once() + + def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None: + scraper = self._make_scraper() + scraper.settings.celery.batch_size = 50 + + page = MagicMock() + empty_page_result = SimpleNamespace( + page_number=1, + vehicle_links=[], + next_page_detected=True, + ) + + scraper._open_listing_for_stream = MagicMock(return_value=( + page, + {"make": None, "model": None, "year_min": None, "year_max": None}, + )) + scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result) + scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, [])) + scraper.sync_batch = MagicMock() + + result = scraper._sync_listing_streaming( + make=None, + model=None, + lane="iaai_cars", + limit=None, + effective_only_new=False, + started_at=0.0, + listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST", + ) + + self.assertEqual(result["total"], 0) + self.assertEqual(result["cars_upserted"], 0) + self.assertEqual(result["cars_failed"], 0) + self.assertEqual(result["listing"]["pages_collected"], 1) + scraper._recover_empty_listing_page.assert_called_once() + scraper.sync_batch.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py new file mode 100644 index 0000000..7d6192a --- /dev/null +++ b/tests/test_scraper.py @@ -0,0 +1,319 @@ +from __future__ import annotations + +import unittest +from concurrent.futures import TimeoutError as FuturesTimeoutError +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from iaai_scraper.core.config import Settings +from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError +from iaai_scraper.scraper import IAAIScraper +from iaai_scraper.storage.schemas import CarRecord + + +def make_db_record(origin_id: str) -> dict[str, object]: + return CarRecord( + parser_id=f"iaai:{origin_id}", + brand="Toyota", + model="Camry", + origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + ).model_dump(mode="json") + + +class TestScraperSync(unittest.TestCase): + def _make_scraper(self) -> IAAIScraper: + s = Settings() + s.log_level = "CRITICAL" + s.database.url = "sqlite://" + return IAAIScraper(s) + + def test_sync_vehicle_uses_db_record(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=1) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) + scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) + + result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") + + self.assertEqual(result["status"], "success") + self.assertIn("trace_id", result) + scraper.persistence.upsert_car.assert_called_once() + + def test_only_new_routing_legacy_and_streaming(self) -> None: + # Legacy path: only_new без listing_url. + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=2) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( + {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"}, + )) + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.iaai.com/VehicleDetail/111~US", + "https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/VehicleDetail/333~US", + ] + }) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], + }) + result = scraper.sync_listing(only_new=True) + self.assertEqual(result["skipped_existing"], 2) + self.assertEqual(result["cars_upserted"], 1) + + # Streaming path: only_new + listing_url. + scraper2 = self._make_scraper() + scraper2.persistence.create_tables = MagicMock() + scraper2.persistence.start_sync_run = MagicMock(return_value=6) + scraper2.persistence.finish_sync_run = MagicMock() + scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used")) + scraper2._sync_listing_streaming = MagicMock(return_value={ + "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False}, + "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0, + "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(), + }) + result2 = scraper2.sync_listing( + only_new=True, + listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + year_min=2020, year_max=2027, + ) + self.assertEqual(result2["cars_upserted"], 10) + scraper2._sync_listing_streaming.assert_called_once() + scraper2.collect_listing.assert_not_called() + + def test_segmented_sync_calls_per_segment_and_resumes(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=3) + scraper.persistence.finish_sync_run = MagicMock() + + call_args_log: list[dict] = [] + def _fake_sync_listing(**kwargs): + call_args_log.append(kwargs) + return { + "status": "success", "cars_upserted": 5, "cars_failed": 0, + "images_upserted": 10, "skipped_existing": 0, + "listing": {"vehicles_collected": 50}, "failures": [], + } + + scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing) + segments = [ + {"make": "TOYOTA", "year_min": 2020, "year_max": 2027}, + {"make": "FORD", "year_min": None, "year_max": None}, + {"make": "HONDA", "year_min": None, "year_max": None}, + ] + + # Resume: пропускаем TOYOTA, начинаем сразу с FORD. + result = scraper.sync_listing_segmented( + segments=segments, start_segment=1, + ) + + self.assertEqual(result["segments_completed"], 2) # FORD + HONDA + self.assertEqual(result["cars_upserted"], 10) + # URL содержит бренд. + self.assertIn("FORD", call_args_log[0]["listing_url"]) + self.assertIn("HONDA", call_args_log[1]["listing_url"]) + + def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None: + scraper = self._make_scraper() + scraper.sync_listing = MagicMock(side_effect=[ + { + "status": "failed", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "listing": {"vehicles_collected": 0}, + "failures": [{"vehicle_url": "segment", "error": "resume failed"}], + }, + { + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "listing": {"vehicles_collected": 10}, + "failures": [], + }, + ]) + + result = scraper.sync_listing_segmented( + segments=[ + {"make": "EAGLE", "year_min": None, "year_max": None}, + {"make": "FORD", "year_min": None, "year_max": None}, + ] + ) + + self.assertFalse(result["full_scan_completed"]) + self.assertEqual(result["status"], "partial_success") + + def test_build_segment_listing_url(self) -> None: + base = "https://www.iaai.com/Vehiclelisting/Cars" + self.assertEqual( + IAAIScraper._build_segment_listing_url(base, "TOYOTA"), + "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + ) + self.assertEqual( + IAAIScraper._build_segment_listing_url(base, "LAND ROVER"), + "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER", + ) + self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base) + self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base) + + def test_guard_and_protection_detection(self) -> None: + with self.assertRaises(AntiBotDetectedError): + IAAIScraper._raise_if_blocked_or_incomplete( + {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, + "access_notes": {}, "vehicle_summary": {}}, + "https://www.iaai.com/VehicleDetail/999~US", + ) + with self.assertRaises(SiteStructureChangedError): + IAAIScraper._raise_if_blocked_or_incomplete( + {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {"possible_captcha": False, "possible_antibot": False}, + "vehicle_summary": {}}, + "https://www.iaai.com/VehicleDetail/999~US", + ) + self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) + self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) + self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) + + def test_close_resets_browser_state(self) -> None: + scraper = self._make_scraper() + http_pool = MagicMock() + scraper._http_pool = http_pool + scraper.context = MagicMock() + scraper.browser = MagicMock() + scraper.playwright = MagicMock() + scraper.close() + http_pool.clear.assert_called_once() + self.assertIsNone(scraper.context) + self.assertIsNone(scraper.browser) + + def test_recover_empty_listing_page(self) -> None: + scraper = self._make_scraper() + page = MagicMock() + page_result = SimpleNamespace( + vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")], + ) + scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) + scraper.listing_collector.open_cars_listing = MagicMock() + + # Страница > 1: reload. + _, urls = scraper._recover_empty_listing_page( + page, page_number=5, all_raw_urls=[], seen_urls=set(), + ) + page.reload.assert_called_once() + self.assertEqual(len(urls), 1) + + # Страница 1: переоткрытие листинга. + page.reset_mock() + _, urls = scraper._recover_empty_listing_page( + page, page_number=1, all_raw_urls=[], seen_urls=set(), + ) + scraper.listing_collector.open_cars_listing.assert_called_once() + page.reload.assert_not_called() + + def test_sync_listing_always_starts_from_page_one(self) -> None: + scraper = self._make_scraper() + scraper.settings.celery.batch_size = 1 + + page = MagicMock() + page_result = SimpleNamespace( + page_number=1, + vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")], + next_page_detected=False, + ) + + scraper._get_page_with_warmup = MagicMock(return_value=page) + # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться. + scraper._reopen_listing_and_resume = MagicMock( + side_effect=AssertionError("pagination resume must not be used") + ) + scraper.listing_collector.open_cars_listing = MagicMock() + scraper.listing_collector.apply_filters = MagicMock(return_value={ + "make": None, + "model": None, + "year_min": None, + "year_max": None, + }) + scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) + scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"]) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "failures": [], + }) + + result = scraper._sync_listing_streaming( + make=None, + model=None, + lane="iaai_cars", + limit=None, + effective_only_new=False, + started_at=0.0, + listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE", + ) + + scraper.listing_collector.open_cars_listing.assert_called_once() + scraper._reopen_listing_and_resume.assert_not_called() + scraper.sync_batch.assert_called_once() + self.assertEqual(result["cars_upserted"], 1) + self.assertEqual(result["total"], 1) + + def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None: + scraper = self._make_scraper() + scraper.persistence.upsert_cars_batch = MagicMock(return_value={ + "inserted": 1, + "updated": 0, + "images_upserted": 0, + }) + + urls = [ + "https://www.iaai.com/VehicleDetail/111~US", + "https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/VehicleDetail/333~US", + ] + first_record = CarRecord.model_validate(make_db_record("111")) + + class _FakeExecutor: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def map(self, fn, iterable, timeout=None): # noqa: ARG002 + yield 0, first_record + raise FuturesTimeoutError() + + with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ + patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ + patch.object(scraper, "_browser_fallback_parallel", return_value={ + "records": [], + "failures": [], + "cars_failed": 0, + "protection_events": 0, + }) as fallback_mock: + result = scraper.sync_batch(urls) + + self.assertEqual(result["cars_upserted"], 1) + fallback_urls = fallback_mock.call_args.args[0] + self.assertEqual(len(fallback_urls), 2) + self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]}) + self.assertNotIn(urls[0], {u for u, _ in fallback_urls}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_self_heal.py b/tests/test_self_heal.py new file mode 100644 index 0000000..f5c74e1 --- /dev/null +++ b/tests/test_self_heal.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock, patch + +from iaai_scraper.worker import self_heal + + +class TestSelfHeal(unittest.TestCase): + def test_read_last_progress_ts_uses_global_key(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = "1776800000" + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertEqual(ts, 1776800000) + redis_client.scan_iter.assert_not_called() + + def test_read_last_progress_ts_fallbacks_to_task_progress_keys(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + self_heal.GLOBAL_PROGRESS_TS_KEY: None, + "iaai:state:task_progress:a": '{"ts": 100}', + "iaai:state:task_progress:b": '{"ts": 250}', + "iaai:state:task_progress:c": '{"ts": 150}', + }.get(key) + redis_client.scan_iter.return_value = [ + "iaai:state:task_progress:a", + "iaai:state:task_progress:b", + "iaai:state:task_progress:c", + ] + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertEqual(ts, 250) + + def test_read_last_progress_ts_ignores_broken_payloads(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + self_heal.GLOBAL_PROGRESS_TS_KEY: None, + "iaai:state:task_progress:a": "{bad-json}", + "iaai:state:task_progress:b": '{"foo": "bar"}', + }.get(key) + redis_client.scan_iter.return_value = [ + "iaai:state:task_progress:a", + "iaai:state:task_progress:b", + ] + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertIsNone(ts) + + @patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("iaai_scraper.worker.self_heal.os.kill") + @patch("builtins.open") + def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None: + open_mock.return_value.__enter__.return_value.read.return_value = "123" + # SIGTERM -> process alive check (pid,0) -> SIGKILL + kill_mock.side_effect = [None, None, None] + + self_heal._kill_worker_process() + + self.assertEqual(kill_mock.call_args_list[0].args[0], 123) + self.assertEqual(kill_mock.call_args_list[1].args, (123, 0)) + self.assertEqual(kill_mock.call_args_list[2].args[0], 123) + + @patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("iaai_scraper.worker.self_heal.os.kill") + @patch("builtins.open") + def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None: + open_mock.return_value.__enter__.return_value.read.return_value = "123" + kill_mock.side_effect = [None, ProcessLookupError()] + + self_heal._kill_worker_process() + + # Только SIGTERM и проверка существования процесса. + self.assertEqual(len(kill_mock.call_args_list), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..97f4ff6 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.core.utils import deep_find_key +from iaai_scraper.core.config import ( + IAAI_DEFAULT_MAKES, + _LARGE_MAKES, + _YEAR_SPLITS, + ProxyConfig, + build_listing_segments_for_makes, + parse_listing_segments, +) + + +class TestDeepFindKey(unittest.TestCase): + def test_finds_nested_and_respects_depth(self) -> None: + payload = { + "root": { + "target": "a", + "nested": [{"target": "b"}, {"x": 1}], + } + } + self.assertEqual(deep_find_key(payload, {"target"}), ["a", "b"]) + + deep_payload = {"l1": {"l2": {"l3": {"target": "value"}}}} + self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=2), []) + self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"]) + + +class TestParseListingSegments(unittest.TestCase): + def test_empty_and_invalid_return_empty(self) -> None: + self.assertEqual(parse_listing_segments(""), []) + self.assertEqual(parse_listing_segments(" "), []) + self.assertEqual(parse_listing_segments("invalid"), []) + + def test_auto_segments_complete_coverage(self) -> None: + """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр.""" + segs = parse_listing_segments("auto") + + # Точное число сегментов. + expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES)) + self.assertEqual(len(segs), expected) + + # Все бренды из списка присутствуют. + makes_in_segments = {s["make"] for s in segs} + for make in IAAI_DEFAULT_MAKES: + self.assertIn(make, makes_in_segments) + + # Крупные бренды разбиты на 3 сегмента с годами. + toyota = [s for s in segs if s["make"] == "TOYOTA"] + self.assertEqual(len(toyota), 3) + for s in toyota: + self.assertIsNotNone(s["year_min"]) + + # Мелкие бренды без годов. + lexus = [s for s in segs if s["make"] == "LEXUS"] + self.assertEqual(len(lexus), 1) + self.assertIsNone(lexus[0]["year_min"]) + + # Годовые диапазоны покрывают 1950-2027. + years = set() + for yr_min, yr_max in _YEAR_SPLITS: + years.update(range(yr_min, yr_max + 1)) + for year in range(1950, 2027): + self.assertIn(year, years) + + def test_json_input_formats(self) -> None: + # Массив строк. + segs = parse_listing_segments('["toyota", "ford"]') + self.assertEqual(len(segs), 2) + self.assertEqual(segs[0]["make"], "TOYOTA") + + # Массив объектов с годами. + segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]') + self.assertEqual(segs[0]["make"], "BMW") + self.assertEqual(segs[0]["year_min"], 2020) + self.assertEqual(segs[0]["year_max"], 2025) + + def test_build_listing_segments_for_makes(self) -> None: + segs = build_listing_segments_for_makes(["toyota", "Lexus", "TOYOTA", " "]) + + toyota = [s for s in segs if s["make"] == "TOYOTA"] + lexus = [s for s in segs if s["make"] == "LEXUS"] + + self.assertEqual(len(toyota), len(_YEAR_SPLITS)) + self.assertEqual(len(lexus), 1) + self.assertIsNone(lexus[0]["year_min"]) + + +class TestProxyConfig(unittest.TestCase): + def test_requests_proxy_url_includes_encoded_credentials(self) -> None: + cfg = ProxyConfig( + server="http://144.31.139.6:3128", + username="carsproxy", + password="pass@word:with/slash", + ) + + self.assertEqual( + cfg.to_requests_proxy_url(), + "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128", + ) + self.assertEqual( + cfg.to_requests_proxies(), + { + "http": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128", + "https": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128", + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py new file mode 100644 index 0000000..b7e625b --- /dev/null +++ b/tests/test_worker_tasks.py @@ -0,0 +1,615 @@ +from __future__ import annotations + +import json +import os +from dataclasses import replace +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from iaai_scraper.worker import tasks +from iaai_scraper.core.config import Settings, settings as base_settings + + +class TestWorkerTaskLockHelpers(unittest.TestCase): + def test_build_listing_segments_from_runtime_config_brands(self) -> None: + with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as fh: + json.dump({"filters": {"brands": ["Toyota", "Lexus", "Toyota"]}}, fh) + runtime_config_file = fh.name + + settings = Settings(runtime_config_file=runtime_config_file) + settings.listing.listing_segments_json = "runtime" + + segs = tasks._build_listing_segments(settings) + + toyota = [s for s in segs if s["make"] == "TOYOTA"] + lexus = [s for s in segs if s["make"] == "LEXUS"] + self.assertEqual(len(toyota), 3) + self.assertEqual(len(lexus), 1) + self.assertIsNone(lexus[0]["year_min"]) + os.unlink(runtime_config_file) + + def test_lock_acquire_refresh_release(self) -> None: + redis_client = MagicMock() + + # Acquire. + redis_client.set.return_value = True + self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)) + redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) + + # Refresh. + redis_client.eval.return_value = 1 + self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)) + + # Release. + redis_client.eval.reset_mock() + tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") + args = redis_client.eval.call_args[0] + self.assertEqual(args[2], "lock:key") + self.assertEqual(args[3], "owner-token") + + def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=False): + persistence = MagicMock() + get_persistence.return_value = persistence + get_redis.return_value = MagicMock() + + tasks.sync_listing_task.push_request(id="task-123") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + persistence.create_tables.assert_called_once() + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "sync_already_running") + + def test_sync_listing_task_releases_owned_lock(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 7, + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 4, + "skipped_existing": 1, + "elapsed_seconds": 1.25, + }): + persistence = MagicMock() + get_persistence.return_value = persistence + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + stop_event = MagicMock() + heartbeat_thread = MagicMock() + start_heartbeat.return_value = (stop_event, heartbeat_thread) + + tasks.sync_listing_task.push_request(id="task-123") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + stop_event.set.assert_called_once() + heartbeat_thread.join.assert_called_once() + release_lock.assert_called_once() + + def test_clear_orphan_sync_listing_lock(self) -> None: + # Нет запущенных задач → удаляет. + redis_client = MagicMock() + redis_client.get.return_value = "owner-token" + redis_client.ttl.return_value = 120 + celery_app = MagicMock() + inspector = MagicMock() + inspector.active.return_value = {"worker@node": []} + inspector.reserved.return_value = {"worker@node": []} + inspector.scheduled.return_value = {"worker@node": []} + celery_app.control.inspect.return_value = inspector + + self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)) + redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY) + + # Задача активна → не удаляет. + redis_client2 = MagicMock() + redis_client2.get.return_value = "owner-token" + inspector2 = MagicMock() + inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]} + inspector2.reserved.return_value = {"worker@node": []} + inspector2.scheduled.return_value = {"worker@node": []} + celery_app2 = MagicMock() + celery_app2.control.inspect.return_value = inspector2 + + self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2)) + redis_client2.delete.assert_not_called() + + def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \ + patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 9, + "cars_upserted": 3, + "cars_failed": 0, + "images_upserted": 5, + "skipped_existing": 0, + "elapsed_seconds": 2.0, + }): + persistence = MagicMock() + get_persistence.return_value = persistence + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + stop_event = MagicMock() + heartbeat_thread = MagicMock() + start_heartbeat.return_value = (stop_event, heartbeat_thread) + + tasks.sync_listing_task.push_request(id="task-456") + try: + result = tasks.sync_listing_task.run(make="Honda") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + clear_orphan.assert_called_once() + self.assertEqual(acquire_lock.call_count, 2) + release_lock.assert_called_once() + + def test_checkpoint_save_load_roundtrip(self) -> None: + storage: dict[str, str] = {} + + def _fake_set(key, value, ex=None): + storage[key] = value + return True + + redis_client = MagicMock() + redis_client.set.side_effect = _fake_set + redis_client.get.side_effect = lambda key: storage.get(key) + + tasks._save_last_completed_segment(redis_client, 12) + + self.assertEqual(tasks._load_last_completed_segment(redis_client), 12) + self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS) + + def test_load_checkpoint_clears_invalid_payload(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = "{not-a-number" + + self.assertIsNone(tasks._load_last_completed_segment(redis_client)) + redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY) + + def test_load_checkpoint_empty_when_missing(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = None + + self.assertIsNone(tasks._load_last_completed_segment(redis_client)) + redis_client.delete.assert_not_called() + + def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None: + segments = [ + {"make": "ACURA"}, + {"make": "AUDI"}, + {"make": "BMW"}, + {"make": "EAGLE"}, + ] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 11, "status": "success", "full_scan_completed": True, + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-resume-seg") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW). + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2) + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) + release_lock.assert_called_once() + + def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None: + segments = [{"make": "ACURA"}, {"make": "AUDI"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + # Оставшийся чекпоинт не должен использоваться. + redis_client.get.side_effect = lambda key: ( + "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 14, "status": "success", "full_scan_completed": True, + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-792") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) + self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"]) + clear_checkpoint.assert_called() + release_lock.assert_called_once() + + def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None: + segments = [{"make": "ACURA"}, {"make": "AUDI"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 15, "status": "success", "full_scan_completed": True, + "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-beyond") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) + release_lock.assert_called_once() + + def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 13, + "status": "partial_success", + "full_scan_completed": True, + "cars_upserted": 2, + "cars_failed": 1, + "images_upserted": 3, + "skipped_existing": 0, + "elapsed_seconds": 4.0, + "failures": [{"vehicle_url": "v", "error": "e"}], + }), \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + tasks.sync_listing_task.push_request(id="task-791") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "partial_success") + # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт + # до браузерного job + после. Главное — вызов произошёл. + clear_checkpoint.assert_called() + release_lock.assert_called_once() + + def test_try_set_followup_pending_deduplicates(self) -> None: + redis_client = MagicMock() + redis_client.set.side_effect = [True, False] + + self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) + self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) + + def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None: + redis_client = MagicMock() + redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT + + streak, should_enqueue = tasks._bump_bootstrap_failure_streak( + redis_client, + reason="max_retries_exceeded", + ) + + self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT) + self.assertFalse(should_enqueue) + redis_client.expire.assert_called_once_with( + tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, + tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, + ) + + def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None: + redis_client = MagicMock() + redis_client.incr.return_value = 1 + + streak, should_enqueue = tasks._bump_bootstrap_failure_streak( + redis_client, + reason="bootstrap_not_completed", + ) + + self.assertEqual(streak, 1) + self.assertTrue(should_enqueue) + + def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_try_set_followup_pending", return_value=True), \ + patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \ + patch.object(tasks, "_clear_followup_pending") as clear_pending, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 99, + "status": "failed", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [{"vehicle_url": "listing", "error": "bad resume"}], + "listing": {"vehicles_collected": 0}, + }): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-900") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "failed") + bump_streak.assert_called_once() + clear_pending.assert_called() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ + patch.object(tasks, "_release_lock_if_owner"), \ + patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \ + patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-soft-timeout") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "timed_out") + set_pending.assert_called_once() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ + patch.object(tasks, "_release_lock_if_owner"), \ + patch.object(tasks, "_try_set_followup_pending", return_value=True), \ + patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \ + patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 101, + "status": "partial_success", + "full_scan_completed": False, + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 1, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + "listing": {"vehicles_collected": 2}, + }): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-breaker-stop") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "partial_success") + # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly. + self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list)) + self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list)) + clear_checkpoint.assert_called_once() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job") as run_job, \ + patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + run_job.return_value = { + "run_id": 17, + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + } + + tasks.sync_listing_task.push_request(id="task-always-full") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + set_full_scan_done.assert_called_with(redis_client, False) + release_lock.assert_called_once() + + def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 18, + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__enter__.return_value.sync_listing = MagicMock() + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-always-full-resume") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1) + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) + release_lock.assert_called_once() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..e2820a9 --- /dev/null +++ b/uv.lock @@ -0,0 +1,1000 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "alembic" +version = "1.18.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/13/8b084e0f2efb0275a1d534838844926f798bd766566b1375174e2448cd31/alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc", size = 2056725, upload-time = "2026-02-10T16:00:47.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/29/6533c317b74f707ea28f8d633734dbda2119bbadfc61b2f3640ba835d0f7/alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", size = 263893, upload-time = "2026-02-10T16:00:49.997Z" }, +] + +[[package]] +name = "amqp" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + +[[package]] +name = "async-timeout" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, +] + +[[package]] +name = "billiard" +version = "4.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, +] + +[[package]] +name = "celery" +version = "5.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "billiard" }, + { name = "click" }, + { name = "click-didyoumean" }, + { name = "click-plugins" }, + { name = "click-repl" }, + { name = "kombu" }, + { name = "python-dateutil" }, + { name = "tzlocal" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" }, +] + +[[package]] +name = "click" +version = "8.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/57/75/31212c6bf2503fdf920d87fee5d7a86a2e3bcf444984126f13d8e4016804/click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", size = 302856, upload-time = "2026-04-03T19:14:45.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/20/71885d8b97d4f3dde17b1fdb92dbd4908b00541c5a3379787137285f602e/click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d", size = 108379, upload-time = "2026-04-03T19:14:43.505Z" }, +] + +[[package]] +name = "click-didyoumean" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, +] + +[[package]] +name = "click-plugins" +version = "1.1.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, +] + +[[package]] +name = "click-repl" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/e0/70553e3000e345daff267cec284ce4cbf3fc141b6da229ac52775b5428f1/coverage-7.13.5.tar.gz", hash = "sha256:c81f6515c4c40141f83f502b07bbfa5c240ba25bbe73da7b33f1e5b6120ff179", size = 915967, upload-time = "2026-03-17T10:33:18.341Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/37/d24c8f8220ff07b839b2c043ea4903a33b0f455abe673ae3c03bbdb7f212/coverage-7.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66a80c616f80181f4d643b0f9e709d97bcea413ecd9631e1dedc7401c8e6695d", size = 219381, upload-time = "2026-03-17T10:30:14.68Z" }, + { url = "https://files.pythonhosted.org/packages/35/8b/cd129b0ca4afe886a6ce9d183c44d8301acbd4ef248622e7c49a23145605/coverage-7.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:145ede53ccbafb297c1c9287f788d1bc3efd6c900da23bf6931b09eafc931587", size = 219880, upload-time = "2026-03-17T10:30:16.231Z" }, + { url = "https://files.pythonhosted.org/packages/55/2f/e0e5b237bffdb5d6c530ce87cc1d413a5b7d7dfd60fb067ad6d254c35c76/coverage-7.13.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0672854dc733c342fa3e957e0605256d2bf5934feeac328da9e0b5449634a642", size = 250303, upload-time = "2026-03-17T10:30:17.748Z" }, + { url = "https://files.pythonhosted.org/packages/92/be/b1afb692be85b947f3401375851484496134c5554e67e822c35f28bf2fbc/coverage-7.13.5-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ec10e2a42b41c923c2209b846126c6582db5e43a33157e9870ba9fb70dc7854b", size = 252218, upload-time = "2026-03-17T10:30:19.804Z" }, + { url = "https://files.pythonhosted.org/packages/da/69/2f47bb6fa1b8d1e3e5d0c4be8ccb4313c63d742476a619418f85740d597b/coverage-7.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be3d4bbad9d4b037791794ddeedd7d64a56f5933a2c1373e18e9e568b9141686", size = 254326, upload-time = "2026-03-17T10:30:21.321Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d0/79db81da58965bd29dabc8f4ad2a2af70611a57cba9d1ec006f072f30a54/coverage-7.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4d2afbc5cc54d286bfb54541aa50b64cdb07a718227168c87b9e2fb8f25e1743", size = 256267, upload-time = "2026-03-17T10:30:23.094Z" }, + { url = "https://files.pythonhosted.org/packages/e5/32/d0d7cc8168f91ddab44c0ce4806b969df5f5fdfdbb568eaca2dbc2a04936/coverage-7.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3ad050321264c49c2fa67bb599100456fc51d004b82534f379d16445da40fb75", size = 250430, upload-time = "2026-03-17T10:30:25.311Z" }, + { url = "https://files.pythonhosted.org/packages/4d/06/a055311d891ddbe231cd69fdd20ea4be6e3603ffebddf8704b8ca8e10a3c/coverage-7.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7300c8a6d13335b29bb76d7651c66af6bd8658517c43499f110ddc6717bfc209", size = 252017, upload-time = "2026-03-17T10:30:27.284Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f6/d0fd2d21e29a657b5f77a2fe7082e1568158340dceb941954f776dce1b7b/coverage-7.13.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:eb07647a5738b89baab047f14edd18ded523de60f3b30e75c2acc826f79c839a", size = 250080, upload-time = "2026-03-17T10:30:29.481Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ab/0d7fb2efc2e9a5eb7ddcc6e722f834a69b454b7e6e5888c3a8567ecffb31/coverage-7.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9adb6688e3b53adffefd4a52d72cbd8b02602bfb8f74dcd862337182fd4d1a4e", size = 253843, upload-time = "2026-03-17T10:30:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/ba/6f/7467b917bbf5408610178f62a49c0ed4377bb16c1657f689cc61470da8ce/coverage-7.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7c8d4bc913dd70b93488d6c496c77f3aff5ea99a07e36a18f865bca55adef8bd", size = 249802, upload-time = "2026-03-17T10:30:33.358Z" }, + { url = "https://files.pythonhosted.org/packages/75/2c/1172fb689df92135f5bfbbd69fc83017a76d24ea2e2f3a1154007e2fb9f8/coverage-7.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e3c426ffc4cd952f54ee9ffbdd10345709ecc78a3ecfd796a57236bfad0b9b8", size = 250707, upload-time = "2026-03-17T10:30:35.2Z" }, + { url = "https://files.pythonhosted.org/packages/67/21/9ac389377380a07884e3b48ba7a620fcd9dbfaf1d40565facdc6b36ec9ef/coverage-7.13.5-cp311-cp311-win32.whl", hash = "sha256:259b69bb83ad9894c4b25be2528139eecba9a82646ebdda2d9db1ba28424a6bf", size = 221880, upload-time = "2026-03-17T10:30:36.775Z" }, + { url = "https://files.pythonhosted.org/packages/af/7f/4cd8a92531253f9d7c1bbecd9fa1b472907fb54446ca768c59b531248dc5/coverage-7.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:258354455f4e86e3e9d0d17571d522e13b4e1e19bf0f8596bcf9476d61e7d8a9", size = 222816, upload-time = "2026-03-17T10:30:38.891Z" }, + { url = "https://files.pythonhosted.org/packages/12/a6/1d3f6155fb0010ca68eba7fe48ca6c9da7385058b77a95848710ecf189b1/coverage-7.13.5-cp311-cp311-win_arm64.whl", hash = "sha256:bff95879c33ec8da99fc9b6fe345ddb5be6414b41d6d1ad1c8f188d26f36e028", size = 221483, upload-time = "2026-03-17T10:30:40.463Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c3/a396306ba7db865bf96fc1fb3b7fd29bcbf3d829df642e77b13555163cd6/coverage-7.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:460cf0114c5016fa841214ff5564aa4864f11948da9440bc97e21ad1f4ba1e01", size = 219554, upload-time = "2026-03-17T10:30:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/a6/16/a68a19e5384e93f811dccc51034b1fd0b865841c390e3c931dcc4699e035/coverage-7.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e223ce4b4ed47f065bfb123687686512e37629be25cc63728557ae7db261422", size = 219908, upload-time = "2026-03-17T10:30:43.906Z" }, + { url = "https://files.pythonhosted.org/packages/29/72/20b917c6793af3a5ceb7fb9c50033f3ec7865f2911a1416b34a7cfa0813b/coverage-7.13.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6e3370441f4513c6252bf042b9c36d22491142385049243253c7e48398a15a9f", size = 251419, upload-time = "2026-03-17T10:30:45.545Z" }, + { url = "https://files.pythonhosted.org/packages/8c/49/cd14b789536ac6a4778c453c6a2338bc0a2fb60c5a5a41b4008328b9acc1/coverage-7.13.5-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:03ccc709a17a1de074fb1d11f217342fb0d2b1582ed544f554fc9fc3f07e95f5", size = 254159, upload-time = "2026-03-17T10:30:47.204Z" }, + { url = "https://files.pythonhosted.org/packages/9d/00/7b0edcfe64e2ed4c0340dac14a52ad0f4c9bd0b8b5e531af7d55b703db7c/coverage-7.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f4818d065964db3c1c66dc0fbdac5ac692ecbc875555e13374fdbe7eedb4376", size = 255270, upload-time = "2026-03-17T10:30:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/93/89/7ffc4ba0f5d0a55c1e84ea7cee39c9fc06af7b170513d83fbf3bbefce280/coverage-7.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:012d5319e66e9d5a218834642d6c35d265515a62f01157a45bcc036ecf947256", size = 257538, upload-time = "2026-03-17T10:30:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/81/bd/73ddf85f93f7e6fa83e77ccecb6162d9415c79007b4bc124008a4995e4a7/coverage-7.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8dd02af98971bdb956363e4827d34425cb3df19ee550ef92855b0acb9c7ce51c", size = 251821, upload-time = "2026-03-17T10:30:52.5Z" }, + { url = "https://files.pythonhosted.org/packages/a0/81/278aff4e8dec4926a0bcb9486320752811f543a3ce5b602cc7a29978d073/coverage-7.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f08fd75c50a760c7eb068ae823777268daaf16a80b918fa58eea888f8e3919f5", size = 253191, upload-time = "2026-03-17T10:30:54.543Z" }, + { url = "https://files.pythonhosted.org/packages/70/ee/fe1621488e2e0a58d7e94c4800f0d96f79671553488d401a612bebae324b/coverage-7.13.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:843ea8643cf967d1ac7e8ecd4bb00c99135adf4816c0c0593fdcc47b597fcf09", size = 251337, upload-time = "2026-03-17T10:30:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/37/a6/f79fb37aa104b562207cc23cb5711ab6793608e246cae1e93f26b2236ed9/coverage-7.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:9d44d7aa963820b1b971dbecd90bfe5fe8f81cff79787eb6cca15750bd2f79b9", size = 255404, upload-time = "2026-03-17T10:30:58.427Z" }, + { url = "https://files.pythonhosted.org/packages/75/f0/ed15262a58ec81ce457ceb717b7f78752a1713556b19081b76e90896e8d4/coverage-7.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7132bed4bd7b836200c591410ae7d97bf7ae8be6fc87d160b2bd881df929e7bf", size = 250903, upload-time = "2026-03-17T10:31:00.093Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e9/9129958f20e7e9d4d56d51d42ccf708d15cac355ff4ac6e736e97a9393d2/coverage-7.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a698e363641b98843c517817db75373c83254781426e94ada3197cabbc2c919c", size = 252780, upload-time = "2026-03-17T10:31:01.916Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d7/0ad9b15812d81272db94379fe4c6df8fd17781cc7671fdfa30c76ba5ff7b/coverage-7.13.5-cp312-cp312-win32.whl", hash = "sha256:bdba0a6b8812e8c7df002d908a9a2ea3c36e92611b5708633c50869e6d922fdf", size = 222093, upload-time = "2026-03-17T10:31:03.642Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/821a9a5799fac2556bcf0bd37a70d1d11fa9e49784b6d22e92e8b2f85f18/coverage-7.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:d2c87e0c473a10bffe991502eac389220533024c8082ec1ce849f4218dded810", size = 222900, upload-time = "2026-03-17T10:31:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/2238c2ad08e35cf4f020ea721f717e09ec3152aea75d191a7faf3ef009a8/coverage-7.13.5-cp312-cp312-win_arm64.whl", hash = "sha256:bf69236a9a81bdca3bff53796237aab096cdbf8d78a66ad61e992d9dac7eb2de", size = 221515, upload-time = "2026-03-17T10:31:07.293Z" }, + { url = "https://files.pythonhosted.org/packages/74/8c/74fedc9663dcf168b0a059d4ea756ecae4da77a489048f94b5f512a8d0b3/coverage-7.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ec4af212df513e399cf11610cc27063f1586419e814755ab362e50a85ea69c1", size = 219576, upload-time = "2026-03-17T10:31:09.045Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c9/44fb661c55062f0818a6ffd2685c67aa30816200d5f2817543717d4b92eb/coverage-7.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:941617e518602e2d64942c88ec8499f7fbd49d3f6c4327d3a71d43a1973032f3", size = 219942, upload-time = "2026-03-17T10:31:10.708Z" }, + { url = "https://files.pythonhosted.org/packages/5f/13/93419671cee82b780bab7ea96b67c8ef448f5f295f36bf5031154ec9a790/coverage-7.13.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:da305e9937617ee95c2e39d8ff9f040e0487cbf1ac174f777ed5eddd7a7c1f26", size = 250935, upload-time = "2026-03-17T10:31:12.392Z" }, + { url = "https://files.pythonhosted.org/packages/ac/68/1666e3a4462f8202d836920114fa7a5ee9275d1fa45366d336c551a162dd/coverage-7.13.5-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:78e696e1cc714e57e8b25760b33a8b1026b7048d270140d25dafe1b0a1ee05a3", size = 253541, upload-time = "2026-03-17T10:31:14.247Z" }, + { url = "https://files.pythonhosted.org/packages/4e/5e/3ee3b835647be646dcf3c65a7c6c18f87c27326a858f72ab22c12730773d/coverage-7.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02ca0eed225b2ff301c474aeeeae27d26e2537942aa0f87491d3e147e784a82b", size = 254780, upload-time = "2026-03-17T10:31:16.193Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/cb5bd1a04cfcc49ede6cd8409d80bee17661167686741e041abc7ee1b9a9/coverage-7.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:04690832cbea4e4663d9149e05dba142546ca05cb1848816760e7f58285c970a", size = 256912, upload-time = "2026-03-17T10:31:17.89Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/c1dceb7b9714473800b075f5c8a84f4588f887a90eb8645282031676e242/coverage-7.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0590e44dd2745c696a778f7bab6aa95256de2cbc8b8cff4f7db8ff09813d6969", size = 251165, upload-time = "2026-03-17T10:31:19.605Z" }, + { url = "https://files.pythonhosted.org/packages/b7/62/5502b73b97aa2e53ea22a39cf8649ff44827bef76d90bf638777daa27a9d/coverage-7.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d7cfad2d6d81dd298ab6b89fe72c3b7b05ec7544bdda3b707ddaecff8d25c161", size = 252908, upload-time = "2026-03-17T10:31:21.312Z" }, + { url = "https://files.pythonhosted.org/packages/7d/37/7792c2d69854397ca77a55c4646e5897c467928b0e27f2d235d83b5d08c6/coverage-7.13.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:e092b9499de38ae0fbfbc603a74660eb6ff3e869e507b50d85a13b6db9863e15", size = 250873, upload-time = "2026-03-17T10:31:23.565Z" }, + { url = "https://files.pythonhosted.org/packages/a3/23/bc866fb6163be52a8a9e5d708ba0d3b1283c12158cefca0a8bbb6e247a43/coverage-7.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:48c39bc4a04d983a54a705a6389512883d4a3b9862991b3617d547940e9f52b1", size = 255030, upload-time = "2026-03-17T10:31:25.58Z" }, + { url = "https://files.pythonhosted.org/packages/7d/8b/ef67e1c222ef49860701d346b8bbb70881bef283bd5f6cbba68a39a086c7/coverage-7.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2d3807015f138ffea1ed9afeeb8624fd781703f2858b62a8dd8da5a0994c57b6", size = 250694, upload-time = "2026-03-17T10:31:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/46/0d/866d1f74f0acddbb906db212e096dee77a8e2158ca5e6bb44729f9d93298/coverage-7.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ee2aa19e03161671ec964004fb74b2257805d9710bf14a5c704558b9d8dbaf17", size = 252469, upload-time = "2026-03-17T10:31:29.472Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f5/be742fec31118f02ce42b21c6af187ad6a344fed546b56ca60caacc6a9a0/coverage-7.13.5-cp313-cp313-win32.whl", hash = "sha256:ce1998c0483007608c8382f4ff50164bfc5bd07a2246dd272aa4043b75e61e85", size = 222112, upload-time = "2026-03-17T10:31:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/7732d648ab9d069a46e686043241f01206348e2bbf128daea85be4d6414b/coverage-7.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:631efb83f01569670a5e866ceb80fe483e7c159fac6f167e6571522636104a0b", size = 222923, upload-time = "2026-03-17T10:31:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/48/af/fea819c12a095781f6ccd504890aaddaf88b8fab263c4940e82c7b770124/coverage-7.13.5-cp313-cp313-win_arm64.whl", hash = "sha256:f4cd16206ad171cbc2470dbea9103cf9a7607d5fe8c242fdf1edf36174020664", size = 221540, upload-time = "2026-03-17T10:31:35.445Z" }, + { url = "https://files.pythonhosted.org/packages/23/d2/17879af479df7fbbd44bd528a31692a48f6b25055d16482fdf5cdb633805/coverage-7.13.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0428cbef5783ad91fe240f673cc1f76b25e74bbfe1a13115e4aa30d3f538162d", size = 220262, upload-time = "2026-03-17T10:31:37.184Z" }, + { url = "https://files.pythonhosted.org/packages/5b/4c/d20e554f988c8f91d6a02c5118f9abbbf73a8768a3048cb4962230d5743f/coverage-7.13.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e0b216a19534b2427cc201a26c25da4a48633f29a487c61258643e89d28200c0", size = 220617, upload-time = "2026-03-17T10:31:39.245Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/f9f5277b95184f764b24e7231e166dfdb5780a46d408a2ac665969416d61/coverage-7.13.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:972a9cd27894afe4bc2b1480107054e062df08e671df7c2f18c205e805ccd806", size = 261912, upload-time = "2026-03-17T10:31:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f6/7f1ab39393eeb50cfe4747ae8ef0e4fc564b989225aa1152e13a180d74f8/coverage-7.13.5-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4b59148601efcd2bac8c4dbf1f0ad6391693ccf7a74b8205781751637076aee3", size = 263987, upload-time = "2026-03-17T10:31:43.724Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d7/62c084fb489ed9c6fbdf57e006752e7c516ea46fd690e5ed8b8617c7d52e/coverage-7.13.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:505d7083c8b0c87a8fa8c07370c285847c1f77739b22e299ad75a6af6c32c5c9", size = 266416, upload-time = "2026-03-17T10:31:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f6/df63d8660e1a0bff6125947afda112a0502736f470d62ca68b288ea762d8/coverage-7.13.5-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:60365289c3741e4db327e7baff2a4aaacf22f788e80fa4683393891b70a89fbd", size = 267558, upload-time = "2026-03-17T10:31:48.293Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/353ca81d36779bd108f6d384425f7139ac3c58c750dcfaafe5d0bee6436b/coverage-7.13.5-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b88c69c8ef5d4b6fe7dea66d6636056a0f6a7527c440e890cf9259011f5e606", size = 261163, upload-time = "2026-03-17T10:31:50.125Z" }, + { url = "https://files.pythonhosted.org/packages/2c/16/2e79106d5749bcaf3aee6d309123548e3276517cd7851faa8da213bc61bf/coverage-7.13.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5b13955d31d1633cf9376908089b7cebe7d15ddad7aeaabcbe969a595a97e95e", size = 263981, upload-time = "2026-03-17T10:31:51.961Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/c29e0c59ffa6942030ae6f50b88ae49988e7e8da06de7ecdbf49c6d4feae/coverage-7.13.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f70c9ab2595c56f81a89620e22899eea8b212a4041bd728ac6f4a28bf5d3ddd0", size = 261604, upload-time = "2026-03-17T10:31:53.872Z" }, + { url = "https://files.pythonhosted.org/packages/40/48/097cdc3db342f34006a308ab41c3a7c11c3f0d84750d340f45d88a782e00/coverage-7.13.5-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:084b84a8c63e8d6fc7e3931b316a9bcafca1458d753c539db82d31ed20091a87", size = 265321, upload-time = "2026-03-17T10:31:55.997Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/4994af354689e14fd03a75f8ec85a9a68d94e0188bbdab3fc1516b55e512/coverage-7.13.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ad14385487393e386e2ea988b09d62dd42c397662ac2dabc3832d71253eee479", size = 260502, upload-time = "2026-03-17T10:31:58.308Z" }, + { url = "https://files.pythonhosted.org/packages/22/c6/9bb9ef55903e628033560885f5c31aa227e46878118b63ab15dc7ba87797/coverage-7.13.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f2c47b36fe7709a6e83bfadf4eefb90bd25fbe4014d715224c4316f808e59a2", size = 262688, upload-time = "2026-03-17T10:32:00.141Z" }, + { url = "https://files.pythonhosted.org/packages/14/4f/f5df9007e50b15e53e01edea486814783a7f019893733d9e4d6caad75557/coverage-7.13.5-cp313-cp313t-win32.whl", hash = "sha256:67e9bc5449801fad0e5dff329499fb090ba4c5800b86805c80617b4e29809b2a", size = 222788, upload-time = "2026-03-17T10:32:02.246Z" }, + { url = "https://files.pythonhosted.org/packages/e1/98/aa7fccaa97d0f3192bec013c4e6fd6d294a6ed44b640e6bb61f479e00ed5/coverage-7.13.5-cp313-cp313t-win_amd64.whl", hash = "sha256:da86cdcf10d2519e10cabb8ac2de03da1bcb6e4853790b7fbd48523332e3a819", size = 223851, upload-time = "2026-03-17T10:32:04.416Z" }, + { url = "https://files.pythonhosted.org/packages/3d/8b/e5c469f7352651e5f013198e9e21f97510b23de957dd06a84071683b4b60/coverage-7.13.5-cp313-cp313t-win_arm64.whl", hash = "sha256:0ecf12ecb326fe2c339d93fc131816f3a7367d223db37817208905c89bded911", size = 222104, upload-time = "2026-03-17T10:32:06.65Z" }, + { url = "https://files.pythonhosted.org/packages/8e/77/39703f0d1d4b478bfd30191d3c14f53caf596fac00efb3f8f6ee23646439/coverage-7.13.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fbabfaceaeb587e16f7008f7795cd80d20ec548dc7f94fbb0d4ec2e038ce563f", size = 219621, upload-time = "2026-03-17T10:32:08.589Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3e/51dff36d99ae14639a133d9b164d63e628532e2974d8b1edb99dd1ebc733/coverage-7.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9bb2a28101a443669a423b665939381084412b81c3f8c0fcfbac57f4e30b5b8e", size = 219953, upload-time = "2026-03-17T10:32:10.507Z" }, + { url = "https://files.pythonhosted.org/packages/6a/6c/1f1917b01eb647c2f2adc9962bd66c79eb978951cab61bdc1acab3290c07/coverage-7.13.5-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bd3a2fbc1c6cccb3c5106140d87cc6a8715110373ef42b63cf5aea29df8c217a", size = 250992, upload-time = "2026-03-17T10:32:12.41Z" }, + { url = "https://files.pythonhosted.org/packages/22/e5/06b1f88f42a5a99df42ce61208bdec3bddb3d261412874280a19796fc09c/coverage-7.13.5-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6c36ddb64ed9d7e496028d1d00dfec3e428e0aabf4006583bb1839958d280510", size = 253503, upload-time = "2026-03-17T10:32:14.449Z" }, + { url = "https://files.pythonhosted.org/packages/80/28/2a148a51e5907e504fa7b85490277734e6771d8844ebcc48764a15e28155/coverage-7.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:380e8e9084d8eb38db3a9176a1a4f3c0082c3806fa0dc882d1d87abc3c789247", size = 254852, upload-time = "2026-03-17T10:32:16.56Z" }, + { url = "https://files.pythonhosted.org/packages/61/77/50e8d3d85cc0b7ebe09f30f151d670e302c7ff4a1bf6243f71dd8b0981fa/coverage-7.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e808af52a0513762df4d945ea164a24b37f2f518cbe97e03deaa0ee66139b4d6", size = 257161, upload-time = "2026-03-17T10:32:19.004Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c4/b5fd1d4b7bf8d0e75d997afd3925c59ba629fc8616f1b3aae7605132e256/coverage-7.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e301d30dd7e95ae068671d746ba8c34e945a82682e62918e41b2679acd2051a0", size = 251021, upload-time = "2026-03-17T10:32:21.344Z" }, + { url = "https://files.pythonhosted.org/packages/f8/66/6ea21f910e92d69ef0b1c3346ea5922a51bad4446c9126db2ae96ee24c4c/coverage-7.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:800bc829053c80d240a687ceeb927a94fd108bbdc68dfbe505d0d75ab578a882", size = 252858, upload-time = "2026-03-17T10:32:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ea/879c83cb5d61aa2a35fb80e72715e92672daef8191b84911a643f533840c/coverage-7.13.5-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:0b67af5492adb31940ee418a5a655c28e48165da5afab8c7fa6fd72a142f8740", size = 250823, upload-time = "2026-03-17T10:32:25.516Z" }, + { url = "https://files.pythonhosted.org/packages/8a/fb/616d95d3adb88b9803b275580bdeee8bd1b69a886d057652521f83d7322f/coverage-7.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c9136ff29c3a91e25b1d1552b5308e53a1e0653a23e53b6366d7c2dcbbaf8a16", size = 255099, upload-time = "2026-03-17T10:32:27.944Z" }, + { url = "https://files.pythonhosted.org/packages/1c/93/25e6917c90ec1c9a56b0b26f6cad6408e5f13bb6b35d484a0d75c9cf000d/coverage-7.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cff784eef7f0b8f6cb28804fbddcfa99f89efe4cc35fb5627e3ac58f91ed3ac0", size = 250638, upload-time = "2026-03-17T10:32:29.914Z" }, + { url = "https://files.pythonhosted.org/packages/fc/7b/dc1776b0464145a929deed214aef9fb1493f159b59ff3c7eeeedf91eddd0/coverage-7.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:68a4953be99b17ac3c23b6efbc8a38330d99680c9458927491d18700ef23ded0", size = 252295, upload-time = "2026-03-17T10:32:31.981Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fb/99cbbc56a26e07762a2740713f3c8f9f3f3106e3a3dd8cc4474954bccd34/coverage-7.13.5-cp314-cp314-win32.whl", hash = "sha256:35a31f2b1578185fbe6aa2e74cea1b1d0bbf4c552774247d9160d29b80ed56cc", size = 222360, upload-time = "2026-03-17T10:32:34.233Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b7/4758d4f73fb536347cc5e4ad63662f9d60ba9118cb6785e9616b2ce5d7fa/coverage-7.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:2aa055ae1857258f9e0045be26a6d62bdb47a72448b62d7b55f4820f361a2633", size = 223174, upload-time = "2026-03-17T10:32:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f2/24d84e1dfe70f8ac9fdf30d338239860d0d1d5da0bda528959d0ebc9da28/coverage-7.13.5-cp314-cp314-win_arm64.whl", hash = "sha256:1b11eef33edeae9d142f9b4358edb76273b3bfd30bc3df9a4f95d0e49caf94e8", size = 221739, upload-time = "2026-03-17T10:32:38.736Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/4a168591057b3668c2428bff25dd3ebc21b629d666d90bcdfa0217940e84/coverage-7.13.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10a0c37f0b646eaff7cce1874c31d1f1ccb297688d4c747291f4f4c70741cc8b", size = 220351, upload-time = "2026-03-17T10:32:41.196Z" }, + { url = "https://files.pythonhosted.org/packages/f5/21/1fd5c4dbfe4a58b6b99649125635df46decdfd4a784c3cd6d410d303e370/coverage-7.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b5db73ba3c41c7008037fa731ad5459fc3944cb7452fc0aa9f822ad3533c583c", size = 220612, upload-time = "2026-03-17T10:32:43.204Z" }, + { url = "https://files.pythonhosted.org/packages/d6/fe/2a924b3055a5e7e4512655a9d4609781b0d62334fa0140c3e742926834e2/coverage-7.13.5-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:750db93a81e3e5a9831b534be7b1229df848b2e125a604fe6651e48aa070e5f9", size = 261985, upload-time = "2026-03-17T10:32:45.514Z" }, + { url = "https://files.pythonhosted.org/packages/d7/0d/c8928f2bd518c45990fe1a2ab8db42e914ef9b726c975facc4282578c3eb/coverage-7.13.5-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9ddb4f4a5479f2539644be484da179b653273bca1a323947d48ab107b3ed1f29", size = 264107, upload-time = "2026-03-17T10:32:47.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ae/4ae35bbd9a0af9d820362751f0766582833c211224b38665c0f8de3d487f/coverage-7.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8a7a2049c14f413163e2bdabd37e41179b1d1ccb10ffc6ccc4b7a718429c607", size = 266513, upload-time = "2026-03-17T10:32:50.1Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/d326174c55af36f74eac6ae781612d9492f060ce8244b570bb9d50d9d609/coverage-7.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1c85e0b6c05c592ea6d8768a66a254bfb3874b53774b12d4c89c481eb78cb90", size = 267650, upload-time = "2026-03-17T10:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5e/31484d62cbd0eabd3412e30d74386ece4a0837d4f6c3040a653878bfc019/coverage-7.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:777c4d1eff1b67876139d24288aaf1817f6c03d6bae9c5cc8d27b83bcfe38fe3", size = 261089, upload-time = "2026-03-17T10:32:54.544Z" }, + { url = "https://files.pythonhosted.org/packages/e9/d8/49a72d6de146eebb0b7e48cc0f4bc2c0dd858e3d4790ab2b39a2872b62bd/coverage-7.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6697e29b93707167687543480a40f0db8f356e86d9f67ddf2e37e2dfd91a9dab", size = 263982, upload-time = "2026-03-17T10:32:56.803Z" }, + { url = "https://files.pythonhosted.org/packages/06/3b/0351f1bd566e6e4dd39e978efe7958bde1d32f879e85589de147654f57bb/coverage-7.13.5-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:8fdf453a942c3e4d99bd80088141c4c6960bb232c409d9c3558e2dbaa3998562", size = 261579, upload-time = "2026-03-17T10:32:59.466Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ce/796a2a2f4017f554d7810f5c573449b35b1e46788424a548d4d19201b222/coverage-7.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:32ca0c0114c9834a43f045a87dcebd69d108d8ffb666957ea65aa132f50332e2", size = 265316, upload-time = "2026-03-17T10:33:01.847Z" }, + { url = "https://files.pythonhosted.org/packages/3d/16/d5ae91455541d1a78bc90abf495be600588aff8f6db5c8b0dae739fa39c9/coverage-7.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8769751c10f339021e2638cd354e13adeac54004d1941119b2c96fe5276d45ea", size = 260427, upload-time = "2026-03-17T10:33:03.945Z" }, + { url = "https://files.pythonhosted.org/packages/48/11/07f413dba62db21fb3fad5d0de013a50e073cc4e2dc4306e770360f6dfc8/coverage-7.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cec2d83125531bd153175354055cdb7a09987af08a9430bd173c937c6d0fba2a", size = 262745, upload-time = "2026-03-17T10:33:06.285Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/d792371332eb4663115becf4bad47e047d16234b1aff687b1b18c58d60ae/coverage-7.13.5-cp314-cp314t-win32.whl", hash = "sha256:0cd9ed7a8b181775459296e402ca4fb27db1279740a24e93b3b41942ebe4b215", size = 223146, upload-time = "2026-03-17T10:33:08.756Z" }, + { url = "https://files.pythonhosted.org/packages/db/51/37221f59a111dca5e85be7dbf09696323b5b9f13ff65e0641d535ed06ea8/coverage-7.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:301e3b7dfefecaca37c9f1aa6f0049b7d4ab8dd933742b607765d757aca77d43", size = 224254, upload-time = "2026-03-17T10:33:11.174Z" }, + { url = "https://files.pythonhosted.org/packages/54/83/6acacc889de8987441aa7d5adfbdbf33d288dad28704a67e574f1df9bcbb/coverage-7.13.5-cp314-cp314t-win_arm64.whl", hash = "sha256:9dacc2ad679b292709e0f5fc1ac74a6d4d5562e424058962c7bb0c658ad25e45", size = 222276, upload-time = "2026-03-17T10:33:13.466Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ee/a4cf96b8ce1e566ed238f0659ac2d3f007ed1d14b181bcb684e19561a69a/coverage-7.13.5-py3-none-any.whl", hash = "sha256:34b02417cf070e173989b3db962f7ed56d2f644307b2cf9d5a0f258e13084a61", size = 211346, upload-time = "2026-03-17T10:33:15.691Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + +[[package]] +name = "fastapi" +version = "0.135.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/e6/7adb4c5fa231e82c35b8f5741a9f2d055f520c29af5546fd70d3e8e1cd2e/fastapi-0.135.3.tar.gz", hash = "sha256:bd6d7caf1a2bdd8d676843cdcd2287729572a1ef524fc4d65c17ae002a1be654", size = 396524, upload-time = "2026-04-01T16:23:58.188Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/a4/5caa2de7f917a04ada20018eccf60d6cc6145b0199d55ca3711b0fc08312/fastapi-0.135.3-py3-none-any.whl", hash = "sha256:9b0f590c813acd13d0ab43dd8494138eb58e484bfac405db1f3187cfc5810d98", size = 117734, upload-time = "2026-04-01T16:23:59.328Z" }, +] + +[[package]] +name = "greenlet" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/94/a5935717b307d7c71fe877b52b884c6af707d2d2090db118a03fbd799369/greenlet-3.4.0.tar.gz", hash = "sha256:f50a96b64dafd6169e595a5c56c9146ef80333e67d4476a65a9c55f400fc22ff", size = 195913, upload-time = "2026-04-08T17:08:00.863Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/c6/dba32cab7e3a625b011aa5647486e2d28423a48845a2998c126dd69c85e1/greenlet-3.4.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:805bebb4945094acbab757d34d6e1098be6de8966009ab9ca54f06ff492def58", size = 285504, upload-time = "2026-04-08T15:52:14.071Z" }, + { url = "https://files.pythonhosted.org/packages/54/f4/7cb5c2b1feb9a1f50e038be79980dfa969aa91979e5e3a18fdbcfad2c517/greenlet-3.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:439fc2f12b9b512d9dfa681c5afe5f6b3232c708d13e6f02c845e0d9f4c2d8c6", size = 605476, upload-time = "2026-04-08T16:24:37.064Z" }, + { url = "https://files.pythonhosted.org/packages/d6/af/b66ab0b2f9a4c5a867c136bf66d9599f34f21a1bcca26a2884a29c450bd9/greenlet-3.4.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70ed1cb0295bee1df57b63bf7f46b4e56a5c93709eea769c1fec1bb23a95875", size = 618336, upload-time = "2026-04-08T16:30:56.59Z" }, + { url = "https://files.pythonhosted.org/packages/6d/31/56c43d2b5de476f77d36ceeec436328533bff960a4cba9a07616e93063ab/greenlet-3.4.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c5696c42e6bb5cfb7c6ff4453789081c66b9b91f061e5e9367fa15792644e76", size = 625045, upload-time = "2026-04-08T16:40:37.111Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5c/8c5633ece6ba611d64bf2770219a98dd439921d6424e4e8cf16b0ac74ea5/greenlet-3.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c660bce1940a1acae5f51f0a064f1bc785d07ea16efcb4bc708090afc4d69e83", size = 613515, upload-time = "2026-04-08T15:56:32.478Z" }, + { url = "https://files.pythonhosted.org/packages/80/ca/704d4e2c90acb8bdf7ae593f5cbc95f58e82de95cc540fb75631c1054533/greenlet-3.4.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:89995ce5ddcd2896d89615116dd39b9703bfa0c07b583b85b89bf1b5d6eddf81", size = 419745, upload-time = "2026-04-08T16:43:04.022Z" }, + { url = "https://files.pythonhosted.org/packages/a9/df/950d15bca0d90a0e7395eb777903060504cdb509b7b705631e8fb69ff415/greenlet-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee407d4d1ca9dc632265aee1c8732c4a2d60adff848057cdebfe5fe94eb2c8a2", size = 1574623, upload-time = "2026-04-08T16:26:18.596Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e7/0839afab829fcb7333c9ff6d80c040949510055d2d4d63251f0d1c7c804e/greenlet-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:956215d5e355fffa7c021d168728321fd4d31fd730ac609b1653b450f6a4bc71", size = 1639579, upload-time = "2026-04-08T15:57:29.231Z" }, + { url = "https://files.pythonhosted.org/packages/d9/2b/b4482401e9bcaf9f5c97f67ead38db89c19520ff6d0d6699979c6efcc200/greenlet-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cb614ace7c27571270354e9c9f696554d073f8aa9319079dcba466bbdead711", size = 238233, upload-time = "2026-04-08T17:02:54.286Z" }, + { url = "https://files.pythonhosted.org/packages/0c/4d/d8123a4e0bcd583d5cfc8ddae0bbe29c67aab96711be331a7cc935a35966/greenlet-3.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:04403ac74fe295a361f650818de93be11b5038a78f49ccfb64d3b1be8fbf1267", size = 235045, upload-time = "2026-04-08T17:04:05.072Z" }, + { url = "https://files.pythonhosted.org/packages/65/8b/3669ad3b3f247a791b2b4aceb3aa5a31f5f6817bf547e4e1ff712338145a/greenlet-3.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1a54a921561dd9518d31d2d3db4d7f80e589083063ab4d3e2e950756ef809e1a", size = 286902, upload-time = "2026-04-08T15:52:12.138Z" }, + { url = "https://files.pythonhosted.org/packages/38/3e/3c0e19b82900873e2d8469b590a6c4b3dfd2b316d0591f1c26b38a4879a5/greenlet-3.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16dec271460a9a2b154e3b1c2fa1050ce6280878430320e85e08c166772e3f97", size = 606099, upload-time = "2026-04-08T16:24:38.408Z" }, + { url = "https://files.pythonhosted.org/packages/b5/33/99fef65e7754fc76a4ed14794074c38c9ed3394a5bd129d7f61b705f3168/greenlet-3.4.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90036ce224ed6fe75508c1907a77e4540176dcf0744473627785dd519c6f9996", size = 618837, upload-time = "2026-04-08T16:30:58.298Z" }, + { url = "https://files.pythonhosted.org/packages/44/57/eae2cac10421feae6c0987e3dc106c6d86262b1cb379e171b017aba893a6/greenlet-3.4.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6f0def07ec9a71d72315cf26c061aceee53b306c36ed38c35caba952ea1b319d", size = 624901, upload-time = "2026-04-08T16:40:38.981Z" }, + { url = "https://files.pythonhosted.org/packages/36/f7/229f3aed6948faa20e0616a0b8568da22e365ede6a54d7d369058b128afd/greenlet-3.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c4f6b453006efb8310affb2d132832e9bbb4fc01ce6df6b70d810d38f1f6dc", size = 615062, upload-time = "2026-04-08T15:56:33.766Z" }, + { url = "https://files.pythonhosted.org/packages/6a/8a/0e73c9b94f31d1cc257fe79a0eff621674141cdae7d6d00f40de378a1e42/greenlet-3.4.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:0e1254cf0cbaa17b04320c3a78575f29f3c161ef38f59c977108f19ffddaf077", size = 423927, upload-time = "2026-04-08T16:43:05.293Z" }, + { url = "https://files.pythonhosted.org/packages/08/97/d988180011aa40135c46cd0d0cf01dd97f7162bae14139b4a3ef54889ba5/greenlet-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b2d9a138ffa0e306d0e2b72976d2fb10b97e690d40ab36a472acaab0838e2de", size = 1573511, upload-time = "2026-04-08T16:26:20.058Z" }, + { url = "https://files.pythonhosted.org/packages/d4/0f/a5a26fe152fb3d12e6a474181f6e9848283504d0afd095f353d85726374b/greenlet-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8424683caf46eb0eb6f626cb95e008e8cc30d0cb675bdfa48200925c79b38a08", size = 1640396, upload-time = "2026-04-08T15:57:30.88Z" }, + { url = "https://files.pythonhosted.org/packages/42/cf/bb2c32d9a100e36ee9f6e38fad6b1e082b8184010cb06259b49e1266ca01/greenlet-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0a53fb071531d003b075c444014ff8f8b1a9898d36bb88abd9ac7b3524648a2", size = 238892, upload-time = "2026-04-08T17:03:10.094Z" }, + { url = "https://files.pythonhosted.org/packages/b7/47/6c41314bac56e71436ce551c7fbe3cc830ed857e6aa9708dbb9c65142eb6/greenlet-3.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:f38b81880ba28f232f1f675893a39cf7b6db25b31cc0a09bb50787ecf957e85e", size = 235599, upload-time = "2026-04-08T15:52:54.3Z" }, + { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" }, + { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" }, + { url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" }, + { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" }, + { url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" }, + { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" }, + { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" }, + { url = "https://files.pythonhosted.org/packages/9b/96/795619651d39c7fbd809a522f881aa6f0ead504cc8201c3a5b789dfaef99/greenlet-3.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:9390ad88b652b1903814eaabd629ca184db15e0eeb6fe8a390bbf8b9106ae15a", size = 235498, upload-time = "2026-04-08T17:05:00.584Z" }, + { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" }, + { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" }, + { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" }, + { url = "https://files.pythonhosted.org/packages/d2/4a/74078d3936712cff6d3c91a930016f476ce4198d84e224fe6d81d3e02880/greenlet-3.4.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:06c2d3b89e0c62ba50bd7adf491b14f39da9e7e701647cb7b9ff4c99bee04b19", size = 673405, upload-time = "2026-04-08T16:40:42.527Z" }, + { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" }, + { url = "https://files.pythonhosted.org/packages/79/3e/df8a83ab894751bc31e1106fdfaa80ca9753222f106b04de93faaa55feb7/greenlet-3.4.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:070b8bac2ff3b4d9e0ff36a0d19e42103331d9737e8504747cd1e659f76297bd", size = 471670, upload-time = "2026-04-08T16:43:08.512Z" }, + { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" }, + { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" }, + { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" }, + { url = "https://files.pythonhosted.org/packages/71/c4/6f621023364d7e85a4769c014c8982f98053246d142420e0328980933ceb/greenlet-3.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:f8296d4e2b92af34ebde81085a01690f26a51eb9ac09a0fcadb331eb36dbc802", size = 236932, upload-time = "2026-04-08T17:04:33.551Z" }, + { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" }, + { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" }, + { url = "https://files.pythonhosted.org/packages/ef/92/466b0d9afd44b8af623139a3599d651c7564fa4152f25f117e1ee5949ffb/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4cd56a9eb7a6444edbc19062f7b6fbc8f287c663b946e3171d899693b1c19fa", size = 665872, upload-time = "2026-04-08T16:40:43.912Z" }, + { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" }, + { url = "https://files.pythonhosted.org/packages/0d/14/3395a7ef3e260de0325152ddfe19dffb3e49fe10873b94654352b53ad48e/greenlet-3.4.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:1f85f204c4d54134ae850d401fa435c89cd667d5ce9dc567571776b45941af72", size = 489237, upload-time = "2026-04-08T16:43:09.993Z" }, + { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" }, + { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "iaai-scraper" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "alembic" }, + { name = "celery" }, + { name = "fastapi" }, + { name = "playwright" }, + { name = "psycopg2-binary" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "redis" }, + { name = "sqlalchemy" }, + { name = "uvicorn" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", specifier = ">=1.14.0" }, + { name = "celery", specifier = ">=5.4.0" }, + { name = "fastapi", specifier = ">=0.115.0" }, + { name = "playwright", specifier = ">=1.53.0" }, + { name = "psycopg2-binary", specifier = ">=2.9.9" }, + { name = "pydantic", specifier = ">=2.8.2" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, + { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "redis", specifier = ">=5.2.0" }, + { name = "sqlalchemy", specifier = ">=2.0.32" }, + { name = "uvicorn", specifier = ">=0.34.0" }, +] +provides-extras = ["dev"] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "kombu" +version = "5.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "amqp" }, + { name = "packaging" }, + { name = "tzdata" }, + { name = "vine" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, +] + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "playwright" +version = "1.58.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098, upload-time = "2026-01-30T15:09:24.028Z" }, + { url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625, upload-time = "2026-01-30T15:09:27.558Z" }, + { url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098, upload-time = "2026-01-30T15:09:30.461Z" }, + { url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268, upload-time = "2026-01-30T15:09:33.787Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214, upload-time = "2026-01-30T15:09:36.751Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998, upload-time = "2026-01-30T15:09:39.627Z" }, + { url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005, upload-time = "2026-01-30T15:09:42.449Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +] + +[[package]] +name = "psycopg2-binary" +version = "2.9.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/6c/8767aaa597ba424643dc87348c6f1754dd9f48e80fdc1b9f7ca5c3a7c213/psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c", size = 379620, upload-time = "2025-10-10T11:14:48.041Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/ae/8d8266f6dd183ab4d48b95b9674034e1b482a3f8619b33a0d86438694577/psycopg2_binary-2.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0e8480afd62362d0a6a27dd09e4ca2def6fa50ed3a4e7c09165266106b2ffa10", size = 3756452, upload-time = "2025-10-10T11:11:11.583Z" }, + { url = "https://files.pythonhosted.org/packages/4b/34/aa03d327739c1be70e09d01182619aca8ebab5970cd0cfa50dd8b9cec2ac/psycopg2_binary-2.9.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:763c93ef1df3da6d1a90f86ea7f3f806dc06b21c198fa87c3c25504abec9404a", size = 3863957, upload-time = "2025-10-10T11:11:16.932Z" }, + { url = "https://files.pythonhosted.org/packages/48/89/3fdb5902bdab8868bbedc1c6e6023a4e08112ceac5db97fc2012060e0c9a/psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e164359396576a3cc701ba8af4751ae68a07235d7a380c631184a611220d9a4", size = 4410955, upload-time = "2025-10-10T11:11:21.21Z" }, + { url = "https://files.pythonhosted.org/packages/ce/24/e18339c407a13c72b336e0d9013fbbbde77b6fd13e853979019a1269519c/psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d57c9c387660b8893093459738b6abddbb30a7eab058b77b0d0d1c7d521ddfd7", size = 4468007, upload-time = "2025-10-10T11:11:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/91/7e/b8441e831a0f16c159b5381698f9f7f7ed54b77d57bc9c5f99144cc78232/psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2c226ef95eb2250974bf6fa7a842082b31f68385c4f3268370e3f3870e7859ee", size = 4165012, upload-time = "2025-10-10T11:11:29.51Z" }, + { url = "https://files.pythonhosted.org/packages/0d/61/4aa89eeb6d751f05178a13da95516c036e27468c5d4d2509bb1e15341c81/psycopg2_binary-2.9.11-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a311f1edc9967723d3511ea7d2708e2c3592e3405677bf53d5c7246753591fbb", size = 3981881, upload-time = "2025-10-30T02:55:07.332Z" }, + { url = "https://files.pythonhosted.org/packages/76/a1/2f5841cae4c635a9459fe7aca8ed771336e9383b6429e05c01267b0774cf/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ebb415404821b6d1c47353ebe9c8645967a5235e6d88f914147e7fd411419e6f", size = 3650985, upload-time = "2025-10-10T11:11:34.975Z" }, + { url = "https://files.pythonhosted.org/packages/84/74/4defcac9d002bca5709951b975173c8c2fa968e1a95dc713f61b3a8d3b6a/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f07c9c4a5093258a03b28fab9b4f151aa376989e7f35f855088234e656ee6a94", size = 3296039, upload-time = "2025-10-10T11:11:40.432Z" }, + { url = "https://files.pythonhosted.org/packages/6d/c2/782a3c64403d8ce35b5c50e1b684412cf94f171dc18111be8c976abd2de1/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:00ce1830d971f43b667abe4a56e42c1e2d594b32da4802e44a73bacacb25535f", size = 3043477, upload-time = "2025-10-30T02:55:11.182Z" }, + { url = "https://files.pythonhosted.org/packages/c8/31/36a1d8e702aa35c38fc117c2b8be3f182613faa25d794b8aeaab948d4c03/psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cffe9d7697ae7456649617e8bb8d7a45afb71cd13f7ab22af3e5c61f04840908", size = 3345842, upload-time = "2025-10-10T11:11:45.366Z" }, + { url = "https://files.pythonhosted.org/packages/6e/b4/a5375cda5b54cb95ee9b836930fea30ae5a8f14aa97da7821722323d979b/psycopg2_binary-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:304fd7b7f97eef30e91b8f7e720b3db75fee010b520e434ea35ed1ff22501d03", size = 2713894, upload-time = "2025-10-10T11:11:48.775Z" }, + { url = "https://files.pythonhosted.org/packages/d8/91/f870a02f51be4a65987b45a7de4c2e1897dd0d01051e2b559a38fa634e3e/psycopg2_binary-2.9.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:be9b840ac0525a283a96b556616f5b4820e0526addb8dcf6525a0fa162730be4", size = 3756603, upload-time = "2025-10-10T11:11:52.213Z" }, + { url = "https://files.pythonhosted.org/packages/27/fa/cae40e06849b6c9a95eb5c04d419942f00d9eaac8d81626107461e268821/psycopg2_binary-2.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f090b7ddd13ca842ebfe301cd587a76a4cf0913b1e429eb92c1be5dbeb1a19bc", size = 3864509, upload-time = "2025-10-10T11:11:56.452Z" }, + { url = "https://files.pythonhosted.org/packages/2d/75/364847b879eb630b3ac8293798e380e441a957c53657995053c5ec39a316/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ab8905b5dcb05bf3fb22e0cf90e10f469563486ffb6a96569e51f897c750a76a", size = 4411159, upload-time = "2025-10-10T11:12:00.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a0/567f7ea38b6e1c62aafd58375665a547c00c608a471620c0edc364733e13/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf940cd7e7fec19181fdbc29d76911741153d51cab52e5c21165f3262125685e", size = 4468234, upload-time = "2025-10-10T11:12:04.892Z" }, + { url = "https://files.pythonhosted.org/packages/30/da/4e42788fb811bbbfd7b7f045570c062f49e350e1d1f3df056c3fb5763353/psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa0f693d3c68ae925966f0b14b8edda71696608039f4ed61b1fe9ffa468d16db", size = 4166236, upload-time = "2025-10-10T11:12:11.674Z" }, + { url = "https://files.pythonhosted.org/packages/3c/94/c1777c355bc560992af848d98216148be5f1be001af06e06fc49cbded578/psycopg2_binary-2.9.11-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a1cf393f1cdaf6a9b57c0a719a1068ba1069f022a59b8b1fe44b006745b59757", size = 3983083, upload-time = "2025-10-30T02:55:15.73Z" }, + { url = "https://files.pythonhosted.org/packages/bd/42/c9a21edf0e3daa7825ed04a4a8588686c6c14904344344a039556d78aa58/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7a6beb4beaa62f88592ccc65df20328029d721db309cb3250b0aae0fa146c3", size = 3652281, upload-time = "2025-10-10T11:12:17.713Z" }, + { url = "https://files.pythonhosted.org/packages/12/22/dedfbcfa97917982301496b6b5e5e6c5531d1f35dd2b488b08d1ebc52482/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:31b32c457a6025e74d233957cc9736742ac5a6cb196c6b68499f6bb51390bd6a", size = 3298010, upload-time = "2025-10-10T11:12:22.671Z" }, + { url = "https://files.pythonhosted.org/packages/66/ea/d3390e6696276078bd01b2ece417deac954dfdd552d2edc3d03204416c0c/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:edcb3aeb11cb4bf13a2af3c53a15b3d612edeb6409047ea0b5d6a21a9d744b34", size = 3044641, upload-time = "2025-10-30T02:55:19.929Z" }, + { url = "https://files.pythonhosted.org/packages/12/9a/0402ded6cbd321da0c0ba7d34dc12b29b14f5764c2fc10750daa38e825fc/psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b6d93d7c0b61a1dd6197d208ab613eb7dcfdcca0a49c42ceb082257991de9d", size = 3347940, upload-time = "2025-10-10T11:12:26.529Z" }, + { url = "https://files.pythonhosted.org/packages/b1/d2/99b55e85832ccde77b211738ff3925a5d73ad183c0b37bcbbe5a8ff04978/psycopg2_binary-2.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:b33fabeb1fde21180479b2d4667e994de7bbf0eec22832ba5d9b5e4cf65b6c6d", size = 2714147, upload-time = "2025-10-10T11:12:29.535Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a8/a2709681b3ac11b0b1786def10006b8995125ba268c9a54bea6f5ae8bd3e/psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8fb3db325435d34235b044b199e56cdf9ff41223a4b9752e8576465170bb38c", size = 3756572, upload-time = "2025-10-10T11:12:32.873Z" }, + { url = "https://files.pythonhosted.org/packages/62/e1/c2b38d256d0dafd32713e9f31982a5b028f4a3651f446be70785f484f472/psycopg2_binary-2.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:366df99e710a2acd90efed3764bb1e28df6c675d33a7fb40df9b7281694432ee", size = 3864529, upload-time = "2025-10-10T11:12:36.791Z" }, + { url = "https://files.pythonhosted.org/packages/11/32/b2ffe8f3853c181e88f0a157c5fb4e383102238d73c52ac6d93a5c8bffe6/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c55b385daa2f92cb64b12ec4536c66954ac53654c7f15a203578da4e78105c0", size = 4411242, upload-time = "2025-10-10T11:12:42.388Z" }, + { url = "https://files.pythonhosted.org/packages/10/04/6ca7477e6160ae258dc96f67c371157776564679aefd247b66f4661501a2/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c0377174bf1dd416993d16edc15357f6eb17ac998244cca19bc67cdc0e2e5766", size = 4468258, upload-time = "2025-10-10T11:12:48.654Z" }, + { url = "https://files.pythonhosted.org/packages/3c/7e/6a1a38f86412df101435809f225d57c1a021307dd0689f7a5e7fe83588b1/psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6ff3335ce08c75afaed19e08699e8aacf95d4a260b495a4a8545244fe2ceb3", size = 4166295, upload-time = "2025-10-10T11:12:52.525Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7d/c07374c501b45f3579a9eb761cbf2604ddef3d96ad48679112c2c5aa9c25/psycopg2_binary-2.9.11-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84011ba3109e06ac412f95399b704d3d6950e386b7994475b231cf61eec2fc1f", size = 3983133, upload-time = "2025-10-30T02:55:24.329Z" }, + { url = "https://files.pythonhosted.org/packages/82/56/993b7104cb8345ad7d4516538ccf8f0d0ac640b1ebd8c754a7b024e76878/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ba34475ceb08cccbdd98f6b46916917ae6eeb92b5ae111df10b544c3a4621dc4", size = 3652383, upload-time = "2025-10-10T11:12:56.387Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ac/eaeb6029362fd8d454a27374d84c6866c82c33bfc24587b4face5a8e43ef/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b31e90fdd0f968c2de3b26ab014314fe814225b6c324f770952f7d38abf17e3c", size = 3298168, upload-time = "2025-10-10T11:13:00.403Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/50c3facc66bded9ada5cbc0de867499a703dc6bca6be03070b4e3b65da6c/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d526864e0f67f74937a8fce859bd56c979f5e2ec57ca7c627f5f1071ef7fee60", size = 3044712, upload-time = "2025-10-30T02:55:27.975Z" }, + { url = "https://files.pythonhosted.org/packages/9c/8e/b7de019a1f562f72ada81081a12823d3c1590bedc48d7d2559410a2763fe/psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04195548662fa544626c8ea0f06561eb6203f1984ba5b4562764fbeb4c3d14b1", size = 3347549, upload-time = "2025-10-10T11:13:03.971Z" }, + { url = "https://files.pythonhosted.org/packages/80/2d/1bb683f64737bbb1f86c82b7359db1eb2be4e2c0c13b947f80efefa7d3e5/psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:efff12b432179443f54e230fdf60de1f6cc726b6c832db8701227d089310e8aa", size = 2714215, upload-time = "2025-10-10T11:13:07.14Z" }, + { url = "https://files.pythonhosted.org/packages/64/12/93ef0098590cf51d9732b4f139533732565704f45bdc1ffa741b7c95fb54/psycopg2_binary-2.9.11-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:92e3b669236327083a2e33ccfa0d320dd01b9803b3e14dd986a4fc54aa00f4e1", size = 3756567, upload-time = "2025-10-10T11:13:11.885Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a9/9d55c614a891288f15ca4b5209b09f0f01e3124056924e17b81b9fa054cc/psycopg2_binary-2.9.11-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e0deeb03da539fa3577fcb0b3f2554a97f7e5477c246098dbb18091a4a01c16f", size = 3864755, upload-time = "2025-10-10T11:13:17.727Z" }, + { url = "https://files.pythonhosted.org/packages/13/1e/98874ce72fd29cbde93209977b196a2edae03f8490d1bd8158e7f1daf3a0/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b52a3f9bb540a3e4ec0f6ba6d31339727b2950c9772850d6545b7eae0b9d7c5", size = 4411646, upload-time = "2025-10-10T11:13:24.432Z" }, + { url = "https://files.pythonhosted.org/packages/5a/bd/a335ce6645334fb8d758cc358810defca14a1d19ffbc8a10bd38a2328565/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:db4fd476874ccfdbb630a54426964959e58da4c61c9feba73e6094d51303d7d8", size = 4468701, upload-time = "2025-10-10T11:13:29.266Z" }, + { url = "https://files.pythonhosted.org/packages/44/d6/c8b4f53f34e295e45709b7568bf9b9407a612ea30387d35eb9fa84f269b4/psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:47f212c1d3be608a12937cc131bd85502954398aaa1320cb4c14421a0ffccf4c", size = 4166293, upload-time = "2025-10-10T11:13:33.336Z" }, + { url = "https://files.pythonhosted.org/packages/4b/e0/f8cc36eadd1b716ab36bb290618a3292e009867e5c97ce4aba908cb99644/psycopg2_binary-2.9.11-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e35b7abae2b0adab776add56111df1735ccc71406e56203515e228a8dc07089f", size = 3983184, upload-time = "2025-10-30T02:55:32.483Z" }, + { url = "https://files.pythonhosted.org/packages/53/3e/2a8fe18a4e61cfb3417da67b6318e12691772c0696d79434184a511906dc/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fcf21be3ce5f5659daefd2b3b3b6e4727b028221ddc94e6c1523425579664747", size = 3652650, upload-time = "2025-10-10T11:13:38.181Z" }, + { url = "https://files.pythonhosted.org/packages/76/36/03801461b31b29fe58d228c24388f999fe814dfc302856e0d17f97d7c54d/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9bd81e64e8de111237737b29d68039b9c813bdf520156af36d26819c9a979e5f", size = 3298663, upload-time = "2025-10-10T11:13:44.878Z" }, + { url = "https://files.pythonhosted.org/packages/97/77/21b0ea2e1a73aa5fa9222b2a6b8ba325c43c3a8d54272839c991f2345656/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:32770a4d666fbdafab017086655bcddab791d7cb260a16679cc5a7338b64343b", size = 3044737, upload-time = "2025-10-30T02:55:35.69Z" }, + { url = "https://files.pythonhosted.org/packages/67/69/f36abe5f118c1dca6d3726ceae164b9356985805480731ac6712a63f24f0/psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3cb3a676873d7506825221045bd70e0427c905b9c8ee8d6acd70cfcbd6e576d", size = 3347643, upload-time = "2025-10-10T11:13:53.499Z" }, + { url = "https://files.pythonhosted.org/packages/e1/36/9c0c326fe3a4227953dfb29f5d0c8ae3b8eb8c1cd2967aa569f50cb3c61f/psycopg2_binary-2.9.11-cp314-cp314-win_amd64.whl", hash = "sha256:4012c9c954dfaccd28f94e84ab9f94e12df76b4afb22331b1f0d3154893a6316", size = 2803913, upload-time = "2025-10-10T11:13:57.058Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, +] + +[[package]] +name = "pyee" +version = "13.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "redis" +version = "7.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/7f/3759b1d0d72b7c92f0d70ffd9dc962b7b7b5ee74e135f9d7d8ab06b8a318/redis-7.4.0.tar.gz", hash = "sha256:64a6ea7bf567ad43c964d2c30d82853f8df927c5c9017766c55a1d1ed95d18ad", size = 4943913, upload-time = "2026-03-24T09:14:37.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/3a/95deec7db1eb53979973ebd156f3369a72732208d1391cd2e5d127062a32/redis-7.4.0-py3-none-any.whl", hash = "sha256:a9c74a5c893a5ef8455a5adb793a31bb70feb821c86eccb62eebef5a19c429ec", size = 409772, upload-time = "2026-03-24T09:14:35.968Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.49" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/45/461788f35e0364a8da7bda51a1fe1b09762d0c32f12f63727998d85a873b/sqlalchemy-2.0.49.tar.gz", hash = "sha256:d15950a57a210e36dd4cec1aac22787e2a4d57ba9318233e2ef8b2daf9ff2d5f", size = 9898221, upload-time = "2026-04-03T16:38:11.704Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/b5/e3617cc67420f8f403efebd7b043128f94775e57e5b84e7255203390ceae/sqlalchemy-2.0.49-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5070135e1b7409c4161133aa525419b0062088ed77c92b1da95366ec5cbebbe", size = 2159126, upload-time = "2026-04-03T16:50:13.242Z" }, + { url = "https://files.pythonhosted.org/packages/20/9b/91ca80403b17cd389622a642699e5f6564096b698e7cdcbcbb6409898bc4/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ac7a3e245fd0310fd31495eb61af772e637bdf7d88ee81e7f10a3f271bff014", size = 3315509, upload-time = "2026-04-03T16:54:49.332Z" }, + { url = "https://files.pythonhosted.org/packages/b1/61/0722511d98c54de95acb327824cb759e8653789af2b1944ab1cc69d32565/sqlalchemy-2.0.49-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d4e5a0ceba319942fa6b585cf82539288a61e314ef006c1209f734551ab9536", size = 3315014, upload-time = "2026-04-03T16:56:56.376Z" }, + { url = "https://files.pythonhosted.org/packages/46/55/d514a653ffeb4cebf4b54c47bec32ee28ad89d39fafba16eeed1d81dccd5/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3ddcb27fb39171de36e207600116ac9dfd4ae46f86c82a9bf3934043e80ebb88", size = 3267388, upload-time = "2026-04-03T16:54:51.272Z" }, + { url = "https://files.pythonhosted.org/packages/2f/16/0dcc56cb6d3335c1671a2258f5d2cb8267c9a2260e27fde53cbfb1b3540a/sqlalchemy-2.0.49-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:32fe6a41ad97302db2931f05bb91abbcc65b5ce4c675cd44b972428dd2947700", size = 3289602, upload-time = "2026-04-03T16:56:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/51/6c/f8ab6fb04470a133cd80608db40aa292e6bae5f162c3a3d4ab19544a67af/sqlalchemy-2.0.49-cp311-cp311-win32.whl", hash = "sha256:46d51518d53edfbe0563662c96954dc8fcace9832332b914375f45a99b77cc9a", size = 2119044, upload-time = "2026-04-03T17:00:53.455Z" }, + { url = "https://files.pythonhosted.org/packages/c4/59/55a6d627d04b6ebb290693681d7683c7da001eddf90b60cfcc41ee907978/sqlalchemy-2.0.49-cp311-cp311-win_amd64.whl", hash = "sha256:951d4a210744813be63019f3df343bf233b7432aadf0db54c75802247330d3af", size = 2143642, upload-time = "2026-04-03T17:00:54.769Z" }, + { url = "https://files.pythonhosted.org/packages/49/b3/2de412451330756aaaa72d27131db6dde23995efe62c941184e15242a5fa/sqlalchemy-2.0.49-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4bbccb45260e4ff1b7db0be80a9025bb1e6698bdb808b83fff0000f7a90b2c0b", size = 2157681, upload-time = "2026-04-03T16:53:07.132Z" }, + { url = "https://files.pythonhosted.org/packages/50/84/b2a56e2105bd11ebf9f0b93abddd748e1a78d592819099359aa98134a8bf/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb37f15714ec2652d574f021d479e78cd4eb9d04396dca36568fdfffb3487982", size = 3338976, upload-time = "2026-04-03T17:07:40Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fa/65fcae2ed62f84ab72cf89536c7c3217a156e71a2c111b1305ab6f0690e2/sqlalchemy-2.0.49-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb9ec6436a820a4c006aad1ac351f12de2f2dbdaad171692ee457a02429b672", size = 3351937, upload-time = "2026-04-03T17:12:23.374Z" }, + { url = "https://files.pythonhosted.org/packages/f8/2f/6fd118563572a7fe475925742eb6b3443b2250e346a0cc27d8d408e73773/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8d6efc136f44a7e8bc8088507eaabbb8c2b55b3dbb63fe102c690da0ddebe55e", size = 3281646, upload-time = "2026-04-03T17:07:41.949Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d7/410f4a007c65275b9cf82354adb4bb8ba587b176d0a6ee99caa16fe638f8/sqlalchemy-2.0.49-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e06e617e3d4fd9e51d385dfe45b077a41e9d1b033a7702551e3278ac597dc750", size = 3316695, upload-time = "2026-04-03T17:12:25.642Z" }, + { url = "https://files.pythonhosted.org/packages/d9/95/81f594aa60ded13273a844539041ccf1e66c5a7bed0a8e27810a3b52d522/sqlalchemy-2.0.49-cp312-cp312-win32.whl", hash = "sha256:83101a6930332b87653886c01d1ee7e294b1fe46a07dd9a2d2b4f91bcc88eec0", size = 2117483, upload-time = "2026-04-03T17:05:40.896Z" }, + { url = "https://files.pythonhosted.org/packages/47/9e/fd90114059175cac64e4fafa9bf3ac20584384d66de40793ae2e2f26f3bb/sqlalchemy-2.0.49-cp312-cp312-win_amd64.whl", hash = "sha256:618a308215b6cececb6240b9abde545e3acdabac7ae3e1d4e666896bf5ba44b4", size = 2144494, upload-time = "2026-04-03T17:05:42.282Z" }, + { url = "https://files.pythonhosted.org/packages/ae/81/81755f50eb2478eaf2049728491d4ea4f416c1eb013338682173259efa09/sqlalchemy-2.0.49-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df2d441bacf97022e81ad047e1597552eb3f83ca8a8f1a1fdd43cd7fe3898120", size = 2154547, upload-time = "2026-04-03T16:53:08.64Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bc/3494270da80811d08bcfa247404292428c4fe16294932bce5593f215cad9/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e20e511dc15265fb433571391ba313e10dd8ea7e509d51686a51313b4ac01a2", size = 3280782, upload-time = "2026-04-03T17:07:43.508Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f5/038741f5e747a5f6ea3e72487211579d8cbea5eb9827a9cbd61d0108c4bd/sqlalchemy-2.0.49-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47604cb2159f8bbd5a1ab48a714557156320f20871ee64d550d8bf2683d980d3", size = 3297156, upload-time = "2026-04-03T17:12:27.697Z" }, + { url = "https://files.pythonhosted.org/packages/88/50/a6af0ff9dc954b43a65ca9b5367334e45d99684c90a3d3413fc19a02d43c/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:22d8798819f86720bc646ab015baff5ea4c971d68121cb36e2ebc2ee43ead2b7", size = 3228832, upload-time = "2026-04-03T17:07:45.38Z" }, + { url = "https://files.pythonhosted.org/packages/bc/d1/5f6bdad8de0bf546fc74370939621396515e0cdb9067402d6ba1b8afbe9a/sqlalchemy-2.0.49-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9b1c058c171b739e7c330760044803099c7fff11511e3ab3573e5327116a9c33", size = 3267000, upload-time = "2026-04-03T17:12:29.657Z" }, + { url = "https://files.pythonhosted.org/packages/f7/30/ad62227b4a9819a5e1c6abff77c0f614fa7c9326e5a3bdbee90f7139382b/sqlalchemy-2.0.49-cp313-cp313-win32.whl", hash = "sha256:a143af2ea6672f2af3f44ed8f9cd020e9cc34c56f0e8db12019d5d9ecf41cb3b", size = 2115641, upload-time = "2026-04-03T17:05:43.989Z" }, + { url = "https://files.pythonhosted.org/packages/17/3a/7215b1b7d6d49dc9a87211be44562077f5f04f9bb5a59552c1c8e2d98173/sqlalchemy-2.0.49-cp313-cp313-win_amd64.whl", hash = "sha256:12b04d1db2663b421fe072d638a138460a51d5a862403295671c4f3987fb9148", size = 2141498, upload-time = "2026-04-03T17:05:45.7Z" }, + { url = "https://files.pythonhosted.org/packages/28/4b/52a0cb2687a9cd1648252bb257be5a1ba2c2ded20ba695c65756a55a15a4/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:24bd94bb301ec672d8f0623eba9226cc90d775d25a0c92b5f8e4965d7f3a1518", size = 3560807, upload-time = "2026-04-03T16:58:31.666Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d8/fda95459204877eed0458550d6c7c64c98cc50c2d8d618026737de9ed41a/sqlalchemy-2.0.49-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a51d3db74ba489266ef55c7a4534eb0b8db9a326553df481c11e5d7660c8364d", size = 3527481, upload-time = "2026-04-03T17:06:00.155Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0a/2aac8b78ac6487240cf7afef8f203ca783e8796002dc0cf65c4ee99ff8bb/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:55250fe61d6ebfd6934a272ee16ef1244e0f16b7af6cd18ab5b1fc9f08631db0", size = 3468565, upload-time = "2026-04-03T16:58:33.414Z" }, + { url = "https://files.pythonhosted.org/packages/a5/3d/ce71cfa82c50a373fd2148b3c870be05027155ce791dc9a5dcf439790b8b/sqlalchemy-2.0.49-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:46796877b47034b559a593d7e4b549aba151dae73f9e78212a3478161c12ab08", size = 3477769, upload-time = "2026-04-03T17:06:02.787Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e8/0a9f5c1f7c6f9ca480319bf57c2d7423f08d31445974167a27d14483c948/sqlalchemy-2.0.49-cp313-cp313t-win32.whl", hash = "sha256:9c4969a86e41454f2858256c39bdfb966a20961e9b58bf8749b65abf447e9a8d", size = 2143319, upload-time = "2026-04-03T17:02:04.328Z" }, + { url = "https://files.pythonhosted.org/packages/0e/51/fb5240729fbec73006e137c4f7a7918ffd583ab08921e6ff81a999d6517a/sqlalchemy-2.0.49-cp313-cp313t-win_amd64.whl", hash = "sha256:b9870d15ef00e4d0559ae10ee5bc71b654d1f20076dbe8bc7ed19b4c0625ceba", size = 2175104, upload-time = "2026-04-03T17:02:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/bf28f618c0a9597d14e0b9ee7d1e0622faff738d44fe986ee287cdf1b8d0/sqlalchemy-2.0.49-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:233088b4b99ebcbc5258c755a097aa52fbf90727a03a5a80781c4b9c54347a2e", size = 2156356, upload-time = "2026-04-03T16:53:09.914Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a7/5f476227576cb8644650eff68cc35fa837d3802b997465c96b8340ced1e2/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57ca426a48eb2c682dae8204cd89ea8ab7031e2675120a47924fabc7caacbc2a", size = 3276486, upload-time = "2026-04-03T17:07:46.9Z" }, + { url = "https://files.pythonhosted.org/packages/2e/84/efc7c0bf3a1c5eef81d397f6fddac855becdbb11cb38ff957888603014a7/sqlalchemy-2.0.49-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685e93e9c8f399b0c96a624799820176312f5ceef958c0f88215af4013d29066", size = 3281479, upload-time = "2026-04-03T17:12:32.226Z" }, + { url = "https://files.pythonhosted.org/packages/91/68/bb406fa4257099c67bd75f3f2261b129c63204b9155de0d450b37f004698/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9e0400fa22f79acc334d9a6b185dc00a44a8e6578aa7e12d0ddcd8434152b187", size = 3226269, upload-time = "2026-04-03T17:07:48.678Z" }, + { url = "https://files.pythonhosted.org/packages/67/84/acb56c00cca9f251f437cb49e718e14f7687505749ea9255d7bd8158a6df/sqlalchemy-2.0.49-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a05977bffe9bffd2229f477fa75eabe3192b1b05f408961d1bebff8d1cd4d401", size = 3248260, upload-time = "2026-04-03T17:12:34.381Z" }, + { url = "https://files.pythonhosted.org/packages/56/19/6a20ea25606d1efd7bd1862149bb2a22d1451c3f851d23d887969201633f/sqlalchemy-2.0.49-cp314-cp314-win32.whl", hash = "sha256:0f2fa354ba106eafff2c14b0cc51f22801d1e8b2e4149342023bd6f0955de5f5", size = 2118463, upload-time = "2026-04-03T17:05:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4f/8297e4ed88e80baa1f5aa3c484a0ee29ef3c69c7582f206c916973b75057/sqlalchemy-2.0.49-cp314-cp314-win_amd64.whl", hash = "sha256:77641d299179c37b89cf2343ca9972c88bb6eef0d5fc504a2f86afd15cd5adf5", size = 2144204, upload-time = "2026-04-03T17:05:48.694Z" }, + { url = "https://files.pythonhosted.org/packages/1f/33/95e7216df810c706e0cd3655a778604bbd319ed4f43333127d465a46862d/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1dc3368794d522f43914e03312202523cc89692f5389c32bea0233924f8d977", size = 3565474, upload-time = "2026-04-03T16:58:35.128Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a4/ed7b18d8ccf7f954a83af6bb73866f5bc6f5636f44c7731fbb741f72cc4f/sqlalchemy-2.0.49-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c821c47ecfe05cc32140dcf8dc6fd5d21971c86dbd56eabfe5ba07a64910c01", size = 3530567, upload-time = "2026-04-03T17:06:04.587Z" }, + { url = "https://files.pythonhosted.org/packages/73/a3/20faa869c7e21a827c4a2a42b41353a54b0f9f5e96df5087629c306df71e/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9c04bff9a5335eb95c6ecf1c117576a0aa560def274876fd156cfe5510fccc61", size = 3474282, upload-time = "2026-04-03T16:58:37.131Z" }, + { url = "https://files.pythonhosted.org/packages/b7/50/276b9a007aa0764304ad467eceb70b04822dc32092492ee5f322d559a4dc/sqlalchemy-2.0.49-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7f605a456948c35260e7b2a39f8952a26f077fd25653c37740ed186b90aaa68a", size = 3480406, upload-time = "2026-04-03T17:06:07.176Z" }, + { url = "https://files.pythonhosted.org/packages/e5/c3/c80fcdb41905a2df650c2a3e0337198b6848876e63d66fe9188ef9003d24/sqlalchemy-2.0.49-cp314-cp314t-win32.whl", hash = "sha256:6270d717b11c5476b0cbb21eedc8d4dbb7d1a956fd6c15a23e96f197a6193158", size = 2149151, upload-time = "2026-04-03T17:02:07.281Z" }, + { url = "https://files.pythonhosted.org/packages/05/52/9f1a62feab6ed368aff068524ff414f26a6daebc7361861035ae00b05530/sqlalchemy-2.0.49-cp314-cp314t-win_amd64.whl", hash = "sha256:275424295f4256fd301744b8f335cff367825d270f155d522b30c7bf49903ee7", size = 2184178, upload-time = "2026-04-03T17:02:08.623Z" }, + { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, +] + +[[package]] +name = "starlette" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/69/17425771797c36cded50b7fe44e850315d039f28b15901ab44839e70b593/starlette-1.0.0.tar.gz", hash = "sha256:6a4beaf1f81bb472fd19ea9b918b50dc3a77a6f2e190a12954b25e6ed5eea149", size = 2655289, upload-time = "2026-03-22T18:29:46.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/c9/584bc9651441b4ba60cc4d557d8a547b5aff901af35bda3a4ee30c819b82/starlette-1.0.0-py3-none-any.whl", hash = "sha256:d3ec55e0bb321692d275455ddfd3df75fff145d009685eb40dc91fc66b03d38b", size = 72651, upload-time = "2026-03-22T18:29:45.111Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/f5/cd531b2d15a671a40c0f66cf06bc3570a12cd56eef98960068ebbad1bf5a/tzdata-2026.1.tar.gz", hash = "sha256:67658a1903c75917309e753fdc349ac0efd8c27db7a0cb406a25be4840f87f98", size = 197639, upload-time = "2026-04-03T11:25:22.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/70/d460bd685a170790ec89317e9bd33047988e4bce507b831f5db771e142de/tzdata-2026.1-py2.py3-none-any.whl", hash = "sha256:4b1d2be7ac37ceafd7327b961aa3a54e467efbdb563a23655fbfe0d39cfc42a9", size = 348952, upload-time = "2026-04-03T11:25:20.313Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/da/6eee1ff8b6cbeed47eeb5229749168e81eb4b7b999a1a15a7176e51410c9/uvicorn-0.44.0.tar.gz", hash = "sha256:6c942071b68f07e178264b9152f1f16dfac5da85880c4ce06366a96d70d4f31e", size = 86947, upload-time = "2026-04-06T09:23:22.826Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/23/a5bbd9600dd607411fa644c06ff4951bec3a4d82c4b852374024359c19c0/uvicorn-0.44.0-py3-none-any.whl", hash = "sha256:ce937c99a2cc70279556967274414c087888e8cec9f9c94644dfca11bd3ced89", size = 69425, upload-time = "2026-04-06T09:23:21.524Z" }, +] + +[[package]] +name = "vine" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, +]