From cccf6b391681f0ad1f9c4aeabd24bf02634dfa8f Mon Sep 17 00:00:00 2001 From: qananasikq Date: Tue, 21 Apr 2026 23:01:33 +0300 Subject: [PATCH] initial openlane project --- .env.example | 93 +- Dockerfile | 10 +- README.md | 314 +-- alembic.ini | 2 +- alembic/env.py | 4 +- alembic/versions/001_initial.py | 2 +- alembic/versions/003_add_composite_indexes.py | 8 +- docker-compose.yml | 76 +- entrypoint.sh | 56 - iaai_scraper/browser/__init__.py | 6 - iaai_scraper/browser/listing.py | 676 ------ iaai_scraper/browser/network.py | 130 - iaai_scraper/browser/pace.py | 46 - iaai_scraper/cli.py | 82 - iaai_scraper/core/config.py | 295 --- iaai_scraper/core/exceptions.py | 14 - iaai_scraper/core/retry.py | 53 - iaai_scraper/core/runtime_config.py | 301 --- iaai_scraper/parsing/mapper.py | 405 ---- iaai_scraper/parsing/parser.py | 408 ---- iaai_scraper/proxy_bridge.py | 317 --- iaai_scraper/scraper.py | 2101 ----------------- iaai_scraper/storage/__init__.py | 1 - iaai_scraper/worker/tasks.py | 648 ----- .../__init__.py | 0 .../api/__init__.py | 0 {iaai_scraper => openlane_scraper}/api/app.py | 4 +- .../api/deps.py | 0 .../api/routes/__init__.py | 0 .../api/routes/cars.py | 0 .../api/routes/health.py | 4 +- .../api/routes/tasks.py | 36 +- openlane_scraper/browser/__init__.py | 3 + .../browser/factory.py | 23 +- openlane_scraper/cli.py | 92 + .../core/__init__.py | 0 openlane_scraper/core/config.py | 194 ++ .../core/logs.py | 0 openlane_scraper/core/runtime_config.py | 379 +++ .../core/utils.py | 5 - openlane_scraper/openlane/__init__.py | 4 + openlane_scraper/openlane/auth.py | 484 ++++ openlane_scraper/openlane/checkpoint.py | 43 + openlane_scraper/openlane/client.py | 376 +++ openlane_scraper/openlane/mapper.py | 424 ++++ openlane_scraper/openlane/runner.py | 273 +++ openlane_scraper/openlane/writer.py | 55 + openlane_scraper/scraper.py | 514 ++++ .../storage}/__init__.py | 0 .../storage/db.py | 16 +- .../storage/enums.py | 1 + .../storage/models.py | 0 .../storage/schemas.py | 0 .../worker/__init__.py | 0 .../worker/celery_app.py | 14 +- openlane_scraper/worker/tasks.py | 360 +++ pyproject.toml | 8 +- runtime_config.json | 2 +- scripts/explore_site.py | 155 ++ tests/test_db.py | 30 +- tests/test_listing.py | 68 - tests/test_mapper.py | 198 ++ tests/test_mappers.py | 94 - tests/test_openlane.py | 129 + tests/test_parser.py | 54 - tests/test_scraper.py | 273 --- tests/test_scraper_images.py | 68 + tests/test_utils.py | 53 +- tests/test_worker_tasks.py | 285 +-- uv.lock | 211 +- 70 files changed, 4176 insertions(+), 6804 deletions(-) delete mode 100644 iaai_scraper/browser/__init__.py delete mode 100644 iaai_scraper/browser/listing.py delete mode 100644 iaai_scraper/browser/network.py delete mode 100644 iaai_scraper/browser/pace.py delete mode 100644 iaai_scraper/cli.py delete mode 100644 iaai_scraper/core/config.py delete mode 100644 iaai_scraper/core/exceptions.py delete mode 100644 iaai_scraper/core/retry.py delete mode 100644 iaai_scraper/core/runtime_config.py delete mode 100644 iaai_scraper/parsing/mapper.py delete mode 100644 iaai_scraper/parsing/parser.py delete mode 100644 iaai_scraper/proxy_bridge.py delete mode 100644 iaai_scraper/scraper.py delete mode 100644 iaai_scraper/storage/__init__.py delete mode 100644 iaai_scraper/worker/tasks.py rename {iaai_scraper => openlane_scraper}/__init__.py (100%) rename {iaai_scraper => openlane_scraper}/api/__init__.py (100%) rename {iaai_scraper => openlane_scraper}/api/app.py (91%) rename {iaai_scraper => openlane_scraper}/api/deps.py (100%) rename {iaai_scraper => openlane_scraper}/api/routes/__init__.py (100%) rename {iaai_scraper => openlane_scraper}/api/routes/cars.py (100%) rename {iaai_scraper => openlane_scraper}/api/routes/health.py (88%) rename {iaai_scraper => openlane_scraper}/api/routes/tasks.py (73%) create mode 100644 openlane_scraper/browser/__init__.py rename {iaai_scraper => openlane_scraper}/browser/factory.py (89%) create mode 100644 openlane_scraper/cli.py rename {iaai_scraper => openlane_scraper}/core/__init__.py (100%) create mode 100644 openlane_scraper/core/config.py rename {iaai_scraper => openlane_scraper}/core/logs.py (100%) create mode 100644 openlane_scraper/core/runtime_config.py rename {iaai_scraper => openlane_scraper}/core/utils.py (93%) create mode 100644 openlane_scraper/openlane/__init__.py create mode 100644 openlane_scraper/openlane/auth.py create mode 100644 openlane_scraper/openlane/checkpoint.py create mode 100644 openlane_scraper/openlane/client.py create mode 100644 openlane_scraper/openlane/mapper.py create mode 100644 openlane_scraper/openlane/runner.py create mode 100644 openlane_scraper/openlane/writer.py create mode 100644 openlane_scraper/scraper.py rename {iaai_scraper/parsing => openlane_scraper/storage}/__init__.py (100%) rename {iaai_scraper => openlane_scraper}/storage/db.py (98%) rename {iaai_scraper => openlane_scraper}/storage/enums.py (97%) rename {iaai_scraper => openlane_scraper}/storage/models.py (100%) rename {iaai_scraper => openlane_scraper}/storage/schemas.py (100%) rename {iaai_scraper => openlane_scraper}/worker/__init__.py (100%) rename {iaai_scraper => openlane_scraper}/worker/celery_app.py (90%) create mode 100644 openlane_scraper/worker/tasks.py create mode 100644 scripts/explore_site.py delete mode 100644 tests/test_listing.py create mode 100644 tests/test_mapper.py delete mode 100644 tests/test_mappers.py create mode 100644 tests/test_openlane.py delete mode 100644 tests/test_parser.py delete mode 100644 tests/test_scraper.py create mode 100644 tests/test_scraper_images.py diff --git a/.env.example b/.env.example index e7c6f52..da46cec 100644 --- a/.env.example +++ b/.env.example @@ -1,52 +1,47 @@ -IAAI_HEADLESS=true -IAAI_LOG_LEVEL=INFO +# --- General --- +OPENLANE_HEADLESS=true +OPENLANE_LOG_LEVEL=INFO +OPENLANE_TIMEOUT_MS=45000 -IAAI_CAPTURE_SAME_ORIGIN_ONLY=true -IAAI_MAX_CAPTURED_REQUESTS=40 -IAAI_MAX_CAPTURED_JSON_RESPONSES=20 +# --- OpenLane Auth & API --- +OPENLANE_USERNAME= +OPENLANE_PASSWORD= +OPENLANE_SIGN_IN_URL=https://app.openlane.com/sign_in +OPENLANE_SEARCH_URL=https://app.openlane.com/search?tab=marketplace +OPENLANE_API_SEARCH_URL=https://app.openlane.com/api/v4/search +OPENLANE_SOURCE_TAB=marketplace +OPENLANE_SALE_TYPES=marketplace +OPENLANE_PAGE_SIZE=30 +OPENLANE_MAX_PAGES=512 +OPENLANE_MAX_CARS_LIMIT=20 +OPENLANE_THROTTLE_MIN_SECONDS=0.3 +OPENLANE_THROTTLE_MAX_SECONDS=0.8 +OPENLANE_RETRY_SCHEDULE_SECONDS=2,5,10 +OPENLANE_REQUEST_TIMEOUT_MS=45000 +OPENLANE_REFRESH_TOKEN= +OPENLANE_OKTA_CLIENT_ID= +OPENLANE_OKTA_TOKEN_ENDPOINT=https://okta.iam.karglobal.com/oauth2/default/v1/token +OPENLANE_OKTA_REDIRECT_URI=https://app.openlane.com/sign_in +OPENLANE_OKTA_TIMEOUT_SECONDS=15 -IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars -IAAI_LISTING_SEGMENTS=auto -IAAI_MAX_PAGES_PER_RUN=999999 -IAAI_MAX_VEHICLES_PER_RUN=999999 -IAAI_PAGE_LINK_LIMIT=999999 -IAAI_INCLUDE_PAGINATION=true -IAAI_COLLECT_CURRENT_PAGE_ONLY=false +# --- Output & State --- +OPENLANE_OUTPUT_DIR=artifacts/openlane +OPENLANE_JSONL_OUTPUT=artifacts/openlane/openlane_search.jsonl +OPENLANE_AGGREGATED_OUTPUT=artifacts/openlane/openlane_search_aggregated.json +OPENLANE_CHECKPOINT_FILE=artifacts/openlane/openlane_checkpoint.json +OPENLANE_STORAGE_STATE_FILE=/data/openlane/openlane_storage_state.json +OPENLANE_PERSIST_STORAGE_STATE=true -IAAI_HUMAN_PACE_ENABLED=true -IAAI_PARALLEL_TABS=10 -CELERY_BATCH_SIZE=100 -IAAI_AFTER_LISTING_OPEN_MIN_S=0.2 -IAAI_AFTER_LISTING_OPEN_MAX_S=0.5 -IAAI_AFTER_FILTER_ACTION_MIN_S=0.2 -IAAI_AFTER_FILTER_ACTION_MAX_S=0.5 -IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02 -IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08 -IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02 -IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08 -IAAI_BETWEEN_VEHICLES_MIN_S=0.02 -IAAI_BETWEEN_VEHICLES_MAX_S=0.08 -IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2 -IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5 +# --- Database (PostgreSQL) --- +OPENLANE_DATABASE_URL=postgresql+psycopg2://openlane:openlane@postgres:5432/openlane_scraper +OPENLANE_DATABASE_ECHO=false +OPENLANE_DATABASE_POOL_SIZE=5 +OPENLANE_DATABASE_MAX_OVERFLOW=5 -IAAI_SYNC_ONLY_NEW=false -IAAI_TOKENS_FILE=/data/tokens.json -IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json - -IAAI_MAX_RETRIES=5 -IAAI_RETRY_DELAY_SECONDS=4 -IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 -IAAI_RETRY_JITTER_SECONDS=0.5 -IAAI_TIMEOUT_MS=90000 -IAAI_FALLBACK_NAV_TIMEOUT_MS=30000 - -IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper -IAAI_DATABASE_ECHO=false -IAAI_DATABASE_POOL_SIZE=5 -IAAI_DATABASE_MAX_OVERFLOW=5 - -IAAI_REDIS_URL=redis://redis:6379/0 +# --- Redis --- +OPENLANE_REDIS_URL=redis://redis:6379/0 +# --- Celery --- CELERY_BROKER_URL=redis://redis:6379/0 CELERY_RESULT_BACKEND=redis://redis:6379/0 CELERY_TASK_SOFT_TIME_LIMIT=86400 @@ -55,3 +50,13 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000 CELERY_WORKER_CONCURRENCY=1 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 CELERY_BEAT_SYNC_LIMIT=0 + +# --- Docker host ports (override if occupied) --- +OPENLANE_API_PORT=58000 +OPENLANE_POSTGRES_PORT=55432 +OPENLANE_REDIS_PORT=56379 + +# --- Proxy (optional) --- +# OPENLANE_PROXY_SERVER=http://127.0.0.1:8899 +# OPENLANE_PROXY_USERNAME= +# OPENLANE_PROXY_PASSWORD= diff --git a/Dockerfile b/Dockerfile index a6f98e3..186dc2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,17 +13,15 @@ RUN pip install --no-cache-dir uv \ && pip install --no-cache-dir -r /tmp/requirements.txt \ && pip install --no-cache-dir playwright-stealth -# Speed-up libs: orjson (fast JSON parse), brotli (HTTP br decompress). -# Pinned outside uv.lock to avoid lock-regen churn. +# orjson + brotli для ускорения JSON/HTTP. RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0" -# Install both Chromium and Firefox. Chromium is the default engine in Docker -# because it works more reliably with the current IAAI listing page. +# Chromium + Firefox. RUN python -m playwright install chromium firefox COPY . . RUN pip install --no-cache-dir -e . -RUN python -m compileall -q iaai_scraper +RUN python -m compileall -q openlane_scraper RUN chmod +x entrypoint.sh RUN chown -R app:app /app @@ -33,4 +31,4 @@ USER app # По умолчанию API, но worker/beat переопределяют CMD в docker-compose ENTRYPOINT ["./entrypoint.sh"] -CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "openlane_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index 14a25db..4592edb 100644 --- a/README.md +++ b/README.md @@ -1,231 +1,138 @@ -# IAAI Scraper +# OpenLane Scraper -Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker. +Парсер автомобилей с [OpenLane marketplace](https://app.openlane.com). Забирает данные через API (`GET /api/v4/search`), сохраняет в PostgreSQL. Работает через Playwright (для авторизации), FastAPI, Celery и Docker. -## Как устроен сайт и его защита +## Архитектура -У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot. +- **Авторизация**: Okta OAuth2 PKCE через `okta.iam.karglobal.com`. Сессия сохраняется в `storage_state.json` и переиспользуется между запусками. +- **Сбор данных**: API-запросы с заголовками `x-rbz-user-id`, `x-rbz-dealership-id` (извлекаются из JWT access_token). +- **Защита аккаунта**: throttle 2–5s между запросами, jitter 0–120s перед стартом, circuit breaker (2 ошибки подряд → стоп), 403→немедленный стоп, 429→backoff×3. +- **Anti-detection**: блокировка ресурсов (bugsnag, intercom, pusher, analytics), stealth patches. -- **Anti-bot** — headless Chromium без прокси часто режется. -- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML. -- **Cookie consent** — при первом заходе показывают баннер. - -Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния. - -## Локальный запуск (без Docker) +## Локальный запуск ### Требования - **Python 3.11+** -- **Git** - -Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта. +- **PostgreSQL** и **Redis** (для Celery) ### Быстрый старт ```bash git clone -cd iaai_scraper_project +cd openlane_scraper_project pip install -e . -playwright install firefox -iaai init-db -``` +playwright install chromium -`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker). +cp .env.example .env +# Заполнить OPENLANE_USERNAME, OPENLANE_PASSWORD в .env -Готовый `.env` уже в репозитории и настроен на SQLite ничего менять не нужно. - -### Запуск парсера - -**Скрапинг одной машины:** - -```bash -iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US" -``` - -**Сбор листинга + скрапинг (например Toyota, 10 штук):** - -```bash -iaai sync-listing --make Toyota --limit 10 -``` - -**Только ссылки с листинга (без скрапинга):** - -```bash -iaai collect-listing --make Toyota -``` - -**С видимым браузером (для отладки):** - -```bash -iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US" -``` - -**Проверка, что записалось в базу (`iaai_scraper.db`):** - -```bash -python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]" -``` - -Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`. - -### Полный стек (API + Worker + Beat) - -Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL: - -```env -IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper -IAAI_REDIS_URL=redis://localhost:6379/0 -CELERY_BROKER_URL=redis://localhost:6379/0 -CELERY_RESULT_BACKEND=redis://localhost:6379/0 -``` - -Применить миграции и запустить в трёх терминалах: - -```bash alembic upgrade head +``` +### Запуск + +```bash # Терминал 1 — API -uvicorn iaai_scraper.api.app:app --reload --port 8000 +uvicorn openlane_scraper.api.app:app --reload --port 8000 # Терминал 2 — Celery Worker -celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping +celery -A openlane_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping -# Терминал 3 — Celery Beat (периодический запуск) -celery -A iaai_scraper.worker.celery_app beat --loglevel=info +# Терминал 3 — Celery Beat (каждый час) +celery -A openlane_scraper.worker.celery_app beat --loglevel=info ``` -API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs` +CLI: ---- +```bash +openlane scrape-openlane --limit 20 +``` -## Запуск через Docker (все сервисы в контейнерах) +## Docker ```bash cp .env.example .env docker compose up -d ``` -Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`. +Сервисы: `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. -В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код. -`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд. +- `beat` запускает сбор **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`) +- Лимит по умолчанию: 20 машин (`OPENLANE_MAX_CARS_LIMIT=20`) -По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`). +API (Docker): `http://localhost:58000` · Swagger: `http://localhost:58000/docs` -Swagger-документация: `http://localhost:8000/docs` +### Первый логин (storage state) ```bash -docker compose ps +docker compose run --rm --service-ports openlane-auth openlane openlane-login +# Войти вручную в браузере — storage state сохранится в volume ``` -- `api` имеет healthcheck на `GET /health` -- `worker` имеет healthcheck через `celery inspect ping` -- `beat` имеет healthcheck по файлу `celerybeat-schedule` - ## API -**Здоровье и статистика:** -- `GET /health` — статус сервиса и подключения к БД -- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов - -**Машины:** +- `GET /health` — статус сервиса +- `GET /api/v1/stats` — статистика (машины, картинки, топ брендов) - `GET /api/v1/cars` — список с пагинацией - `GET /api/v1/cars/{id}` — карточка с картинками -- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number - -**Задачи:** -- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL -- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга +- `POST /api/v1/tasks/sync-listing` — запустить сбор - `GET /api/v1/sync-runs` — история запусков -Скрапим конкретную машину: - -```bash -curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \ - -H "Content-Type: application/json" \ - -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}' -``` - -## CLI - -Для отладки без API и Celery: - -```bash -iaai init-db -iaai collect-listing --make Toyota -iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US" -iaai sync-listing --limit 10 -``` - -`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic. - ## Структура ```text -iaai_scraper/ - scraper.py — оркестратор: связывает browser → parser → storage - cli.py — CLI-команды (init-db, sync-vehicle, sync-listing, ...) - proxy_bridge.py — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией) +openlane_scraper/ + scraper.py — оркестратор: circuit breaker, batched upsert, early-stop + cli.py — CLI-команды (scrape-openlane, openlane-login) browser/ - factory.py — создание браузера, stealth-инъекции, fingerprint - listing.py — сбор ссылок на машины с листинга, пагинация - network.py — перехват XHR/fetch ответов через Playwright events - pace.py — рандомные паузы и движение мыши + factory.py — создание браузера, stealth-инъекции, resource blocking - parsing/ - parser.py — VehicleParser: DOM + XHR JSON + embedded JSON - mapper.py — CarMapper: нормализация → CarRecord + openlane/ + auth.py — авторизация через storage_state / Okta PKCE + client.py — запросы к /api/v4/search с JWT-заголовками + checkpoint.py — checkpoint/resume по страницам + writer.py — JSONL + aggregated JSON + runner.py — orchestration, retry/backoff, progress storage/ models.py — SQLAlchemy: Car, Image, SyncRun schemas.py — Pydantic: CarRecord, ImageRecord, CarRead - enums.py — допустимые значения (drive, gearbox, body_type, ...) + enums.py — enum-значения (drive, gearbox, body_type, ...) db.py — PersistenceService: upsert, sync runs, статистика api/ - app.py — FastAPI factory, lifespan, роутеры - deps.py — dependency injection (Settings, PersistenceService) - routes/ - health.py — GET /health - cars.py — CRUD по машинам + GET /stats - tasks.py — запуск Celery-задач и история sync-runs + app.py — FastAPI factory + deps.py — dependency injection + routes/ — health, cars, tasks worker/ celery_app.py — конфиг Celery, beat-расписание - tasks.py — sync_vehicle_task, sync_listing_task + tasks.py — sync_listing_task с jitter и lock core/ config.py — Settings (dataclass), env-переменные - logs.py — логирование с trace_id (ContextVar) - retry.py — декоратор @retryable с exponential backoff - runtime_config.py — чтение и нормализация runtime-конфига - utils.py — VIN_RE, deep_find_key, вспомогательные функции - -alembic/ — миграции БД -tests/ — тесты + logs.py — логирование с trace_id + retry.py — @retryable с exponential backoff + runtime_config.py — runtime-конфиг + utils.py — утилиты ``` ## Конфигурация Всё через env-переменные (полный список в `.env.example`): -- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE` -- **Redis:** `IAAI_REDIS_URL` -- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES` -- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN` -- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD` -- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д. +- **OpenLane**: `OPENLANE_USERNAME`, `OPENLANE_PASSWORD`, `OPENLANE_MAX_CARS_LIMIT` +- **БД**: `OPENLANE_DATABASE_URL`, `OPENLANE_DATABASE_POOL_SIZE` +- **Redis**: `OPENLANE_REDIS_URL` +- **Celery**: `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES` +- **Прокси**: `OPENLANE_PROXY_SERVER`, `OPENLANE_PROXY_USERNAME`, `OPENLANE_PROXY_PASSWORD` ## Миграции -В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`). - -`api`, `worker`, `beat` стартуют после успешного завершения `migrate`. - -Вручную: +В Docker миграции выполняются сервисом `migrate` (`alembic upgrade head`). ```bash alembic upgrade head @@ -240,105 +147,6 @@ pytest -q Тесты работают на SQLite in-memory, без внешних зависимостей. -## `pyproject.toml` + `uv.lock` - -Проект переведён на современную схему зависимостей: - -- `pyproject.toml` — декларация зависимостей и метаданных проекта -- `uv.lock` — зафиксированные версии для воспроизводимых установок - -Локально можно использовать: - -```bash -uv sync -uv run pytest -q -``` - ## Runtime-фильтры Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей. - -### Секция `sync` - -Поддерживаются поля: - -- `name` -- `ids_initial_size` -- `ids_next_size` -- `ids_max_pages` -- `condition_check_enabled` -- `lane` -- `only_new` -- `limit` - -### Секция `filters` - -Поддерживаются поля: - -- `brands` -- `models` -- `years` -- `body_types` -- `colors` -- `drives` -- `gearboxes` -- `locations` -- `exclude_brands` -- `exclude_models` -- `exclude_years` -- `exclude_body_types` -- `exclude_colors` -- `exclude_drives` -- `exclude_gearboxes` -- `exclude_locations` -- `price.min`, `price.max` -- `mileage.min`, `mileage.max` -- `flags.damaged_only`, `flags.run_and_drive` - -Пример: - -```json -{ - "sync": { - "name": null, - "ids_initial_size": null, - "ids_next_size": null, - "ids_max_pages": null, - "condition_check_enabled": false, - "lane": "iaai_cars", - "only_new": true, - "limit": 50 - }, - "filters": { - "price": { - "min": null, - "max": null - }, - "mileage": { - "min": null, - "max": null - }, - "flags": { - "damaged_only": null, - "run_and_drive": null - }, - "brands": ["Toyota", "Honda"], - "models": [], - "years": [2021, 2022], - "body_types": ["SUV"], - "colors": [], - "drives": [], - "gearboxes": [], - "locations": [], - "exclude_brands": [], - "exclude_models": [], - "exclude_years": [], - "exclude_body_types": [] - } -} -``` - -Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`. - -CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр. - diff --git a/alembic.ini b/alembic.ini index f4be675..4ac416e 100644 --- a/alembic.ini +++ b/alembic.ini @@ -3,7 +3,7 @@ [alembic] script_location = alembic prepend_sys_path = . -sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper +sqlalchemy.url = postgresql+psycopg2://openlane:openlane@localhost:5432/openlane_scraper [loggers] keys = root,sqlalchemy,alembic diff --git a/alembic/env.py b/alembic/env.py index f6f4ab0..1b1282b 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool # Добавляем корень проекта в sys.path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -from iaai_scraper.core.config import Settings -from iaai_scraper.storage.models import Base +from openlane_scraper.core.config import Settings +from openlane_scraper.storage.models import Base config = context.config diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py index 4179d3e..b8f9cea 100644 --- a/alembic/versions/001_initial.py +++ b/alembic/versions/001_initial.py @@ -11,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # Таблица cars — аукционные машины с IAAI + # Таблица cars op.create_table( "cars", sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), diff --git a/alembic/versions/003_add_composite_indexes.py b/alembic/versions/003_add_composite_indexes.py index 1388d45..118494d 100644 --- a/alembic/versions/003_add_composite_indexes.py +++ b/alembic/versions/003_add_composite_indexes.py @@ -10,12 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # Partial index для активных машин IAAI (is_sold = FALSE) + # Partial index для активных машин OpenLane (is_sold = FALSE) # Ускоряет поиск при mark_sold операциях op.execute( - "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai " + "CREATE INDEX IF NOT EXISTS ix_cars_active_openlane " "ON cars (origin_url) " - "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'" + "WHERE is_sold = FALSE AND origin_id LIKE 'openlane:%'" ) # Composite index для проверки дубликатов изображений @@ -35,4 +35,4 @@ def upgrade() -> None: def downgrade() -> None: op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id") op.drop_index("ix_images_car_id_fullres", table_name="images") - op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai") + op.execute("DROP INDEX IF EXISTS ix_cars_active_openlane") diff --git a/docker-compose.yml b/docker-compose.yml index b7f1585..aed48b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,27 +1,24 @@ x-app-env: &app-env - # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here - IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper - IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0} + # PostgreSQL — единственная БД в Docker + OPENLANE_DATABASE_URL: postgresql+psycopg2://openlane:openlane@postgres:5432/openlane_scraper + OPENLANE_REDIS_URL: ${OPENLANE_REDIS_URL:-redis://redis:6379/0} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} - IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800} + OPENLANE_DATABASE_POOL_RECYCLE_SECONDS: ${OPENLANE_DATABASE_POOL_RECYCLE_SECONDS:-1800} CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} - # 0 => без лимита (в коде интерпретируется как None). - # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто. CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3} - CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} - IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40} - IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true} - IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto} - IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999} - IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000} - IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true} - IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json} - IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} - IAAI_BROWSER_ENGINE: chromium + OPENLANE_MAX_CARS_LIMIT: ${OPENLANE_MAX_CARS_LIMIT:-20} + OPENLANE_THROTTLE_MIN_SECONDS: ${OPENLANE_THROTTLE_MIN_SECONDS:-2.0} + OPENLANE_THROTTLE_MAX_SECONDS: ${OPENLANE_THROTTLE_MAX_SECONDS:-5.0} + OPENLANE_CONCURRENCY: ${OPENLANE_CONCURRENCY:-1} + OPENLANE_MAX_PAGES: ${OPENLANE_MAX_PAGES:-512} + OPENLANE_BROWSER_ENGINE: chromium + OPENLANE_HEADLESS: "true" + OPENLANE_STORAGE_STATE_FILE: /data/openlane/openlane_storage_state.json + OPENLANE_RUNTIME_CONFIG_FILE: /app/runtime_config.json TZ: ${TZ:-UTC} x-env-file: &env-file @@ -40,23 +37,24 @@ x-worker-service: &worker-service volumes: - ./runtime_config.json:/app/runtime_config.json:ro - tokens_data:/data + - openlane_data:/data/openlane services: # PostgreSQL postgres: image: postgres:16-alpine - container_name: iaai-postgres + container_name: openlane-postgres restart: unless-stopped environment: - POSTGRES_USER: iaai - POSTGRES_PASSWORD: iaai - POSTGRES_DB: iaai_scraper + POSTGRES_USER: openlane + POSTGRES_PASSWORD: openlane + POSTGRES_DB: openlane_scraper ports: - - "127.0.0.1:5432:5432" + - "127.0.0.1:${OPENLANE_POSTGRES_PORT:-55432}:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"] + test: ["CMD-SHELL", "pg_isready -U openlane -d openlane_scraper"] interval: 5s timeout: 3s retries: 5 @@ -69,10 +67,10 @@ services: # Redis (Celery broker) redis: image: redis:7-alpine - container_name: iaai-redis + container_name: openlane-redis restart: unless-stopped ports: - - "127.0.0.1:6379:6379" + - "127.0.0.1:${OPENLANE_REDIS_PORT:-56379}:6379" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s @@ -93,7 +91,7 @@ services: # DB migrations migrate: <<: *app-service - container_name: iaai-migrate + container_name: openlane-migrate restart: "no" depends_on: postgres: @@ -103,17 +101,17 @@ services: # FastAPI api: <<: *app-service - container_name: iaai-api + container_name: openlane-api restart: unless-stopped ports: - - "127.0.0.1:8000:8000" + - "127.0.0.1:${OPENLANE_API_PORT:-58000}:8000" depends_on: migrate: condition: service_completed_successfully redis: condition: service_healthy command: > - uvicorn iaai_scraper.api.app:app + uvicorn openlane_scraper.api.app:app --host 0.0.0.0 --port 8000 --workers 1 healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] @@ -131,11 +129,11 @@ services: # Celery Worker worker: <<: *worker-service - container_name: iaai-worker + container_name: openlane-worker restart: unless-stopped init: true - mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g} - memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g} + mem_limit: ${OPENLANE_WORKER_MEM_LIMIT:-2g} + memswap_limit: ${OPENLANE_WORKER_MEM_LIMIT:-2g} depends_on: migrate: condition: service_completed_successfully @@ -143,7 +141,7 @@ services: condition: service_healthy stop_grace_period: 60s command: > - celery -A iaai_scraper.worker.celery_app worker + celery -A openlane_scraper.worker.celery_app worker --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork --pidfile=/tmp/celery-worker.pid -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3} @@ -162,7 +160,7 @@ services: # Celery Beat (периодический планировщик) beat: <<: *worker-service - container_name: iaai-beat + container_name: openlane-beat restart: unless-stopped init: true depends_on: @@ -171,7 +169,7 @@ services: redis: condition: service_healthy command: > - celery -A iaai_scraper.worker.celery_app beat + celery -A openlane_scraper.worker.celery_app beat --loglevel=info --pidfile=/tmp/celery-beat.pid --schedule=/tmp/celerybeat-schedule @@ -187,7 +185,17 @@ services: max-size: "10m" max-file: "5" + openlane-auth: + <<: *worker-service + container_name: openlane-auth + profiles: ["auth"] + stdin_open: true + tty: true + command: > + python scripts/explore_site.py + volumes: pgdata: redisdata: tokens_data: + openlane_data: diff --git a/entrypoint.sh b/entrypoint.sh index 078cf3f..bdc74b0 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,60 +1,4 @@ #!/bin/bash set -euo pipefail -is_worker_command() { - local joined="$*" - case "$joined" in - *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*) - return 0 - ;; - esac - return 1 -} - -is_scrape_cli_command() { - local joined="$*" - case "$joined" in - *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*) - return 0 - ;; - esac - return 1 -} - -needs_browser_runtime() { - is_worker_command "$@" || is_scrape_cli_command "$@" -} - -start_proxy_bridge_if_needed() { - if ! needs_browser_runtime "$@"; then - return 0 - fi - - if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then - return 0 - fi - - echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..." - - if [ -z "${IAAI_PROXY_SERVER:-}" ]; then - export IAAI_PROXY_SERVER="http://127.0.0.1:8899" - elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then - export IAAI_PROXY_SERVER="http://127.0.0.1:8899" - fi - - echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}" - python -m iaai_scraper.proxy_bridge & - BRIDGE_PID=$! - sleep 1 - - if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then - echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start" - exit 1 - fi - - echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})" -} - -start_proxy_bridge_if_needed "$@" - exec "$@" diff --git a/iaai_scraper/browser/__init__.py b/iaai_scraper/browser/__init__.py deleted file mode 100644 index 38bb3f4..0000000 --- a/iaai_scraper/browser/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .factory import BrowserFactory -from .listing import ListingCollector -from .network import NetworkCapture -from .pace import HumanPacer - -__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"] diff --git a/iaai_scraper/browser/listing.py b/iaai_scraper/browser/listing.py deleted file mode 100644 index 324b25f..0000000 --- a/iaai_scraper/browser/listing.py +++ /dev/null @@ -1,676 +0,0 @@ -import logging -import re -import time -from dataclasses import asdict, dataclass, field -from typing import Any -from urllib.parse import urljoin - -from playwright.sync_api import Page - -from .pace import HumanPacer -from ..core.config import Settings -from ..core.utils import first_non_empty - -logger = logging.getLogger("iaai_scraper.listing") -VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) -VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']" -COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = ( - "button:has-text('Accept All')", - "button:has-text('Accept all')", - "button:has-text('I Agree')", - "button:has-text('Agree')", - "button:has-text('Only necessary')", - "button:has-text('Только необходимые')", - "button:has-text('Принять все')", - "[id*='accept']", - "[class*='accept']", -) - - -@dataclass(slots=True) -class ListingVehicleLink: - href: str - title: str = "" - lot_number: str | None = None - - -@dataclass(slots=True) -class ListingPageResult: - source_url: str - page_number: int - vehicle_links: list[ListingVehicleLink] = field(default_factory=list) - pagination_available: bool = False - next_page_detected: bool = False - - -class ListingCollector: - _NEXT_PAGE_SELECTORS: tuple[str, ...] = ( - "a[aria-label*='Next']", - "button[aria-label*='Next']", - "a[aria-label*='next']", - "button[aria-label*='next']", - "a[title*='Next']", - "button[title*='Next']", - "a[title*='next']", - "button[title*='next']", - "a[rel='next']", - "link[rel='next']", - "a.pagination-next", - "button.pagination-next", - "a.next", - "button.next", - "a:has-text('Next')", - "button:has-text('Next')", - "a:has-text('NEXT')", - "button:has-text('NEXT')", - "a:has-text('›')", - "button:has-text('›')", - "a:has-text('»')", - "button:has-text('»')", - "a:has(img[src*='icon-arrow-right'])", - "button:has(img[src*='icon-arrow-right'])", - "a:has(img[src*='arrow-right'])", - "button:has(img[src*='arrow-right'])", - ) - - def __init__(self, settings: Settings, pacer: HumanPacer) -> None: - self.settings = settings - self.pacer = pacer - - @staticmethod - def _get_current_page_number(page: Page) -> int | None: - try: - value = page.evaluate( - """ - () => { - const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); - const current = controls.find((el) => { - const text = (el.textContent || '').trim(); - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); - return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); - }); - if (current) { - return parseInt((current.textContent || '').trim(), 10); - } - const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i); - return match ? parseInt(match[1], 10) : null; - } - """ - ) - return int(value) if value is not None else None - except Exception: - return None - - @staticmethod - def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool: - if old_first_href: - try: - page.wait_for_function( - f"""() => {{ - const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\"); - return a && a.getAttribute('href') !== '{old_first_href}'; - }}""", - timeout=12000, - ) - return True - except Exception: - pass - - if expected_page_number is not None: - current_page = ListingCollector._get_current_page_number(page) - if current_page == expected_page_number: - return True - - try: - page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) - except Exception: - pass - - current_page = ListingCollector._get_current_page_number(page) - if expected_page_number is not None and current_page == expected_page_number: - return True - - return not old_first_href - - def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None: - url = url_override or self.settings.listing.cars_url - logger.info("Opening cars listing page: %s", url) - last_err = None - for attempt in range(3): - try: - page.goto(url, wait_until="commit", timeout=60_000) - last_err = None - break - except Exception as e: - last_err = e - logger.warning("goto listing attempt %d failed: %s", attempt + 1, e) - # Небольшой backoff при ошибках открытия листинга. - time.sleep(5 * (attempt + 1)) - if last_err: - logger.warning("All goto attempts failed, trying JS navigation") - try: - page.evaluate(f"window.location.href = '{url}'") - except Exception: - pass - # Быстрая проверка готовности страницы. - try: - page.wait_for_load_state("domcontentloaded", timeout=12_000) - except Exception: - pass - self._accept_cookie_banner(page) - self._wait_for_listing_content(page) - logger.info("Listing page URL: %s", page.url) - self.pacer.after_listing_open() - - def _accept_cookie_banner(self, page: Page) -> None: - for selector in COOKIE_ACCEPT_SELECTORS: - locator = page.locator(selector).first - try: - if locator.count() == 0: - continue - if not locator.is_visible(timeout=500): - continue - locator.click(timeout=2_000) - logger.info("Accepted cookie banner using selector: %s", selector) - try: - page.wait_for_load_state("domcontentloaded", timeout=3_000) - except Exception: - pass - return - except Exception: - continue - - def _wait_for_listing_content(self, page: Page) -> None: - try: - page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000) - return - except Exception: - pass - - # Fallback: React/SSR разметка может появиться не сразу, даже если ещё нет в DOM. - try: - page.wait_for_function( - """() => { - const html = document.documentElement?.innerHTML || ''; - const text = document.body?.innerText || ''; - return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text); - }""", - timeout=12_000, - ) - except Exception: - # Короткая пауза вместо длинного sleep. - time.sleep(1.0) - - def apply_filters( - self, - page: Page, - make: str | None = None, - model: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - ) -> dict[str, str | int | None]: - applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None} - if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): - applied["make"] = make - self.pacer.after_filter_action() - if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): - applied["model"] = model - self.pacer.after_filter_action() - if year_min is not None or year_max is not None: - if self._apply_year_range(page, year_min, year_max): - applied["year_min"] = year_min - applied["year_max"] = year_max - self.pacer.after_filter_action() - return applied - - def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool: - """Заполняет поля фильтра Year и нажимает Apply Year.""" - if year_min is None and year_max is None: - return False - try: - success = page.evaluate( - """([yearMin, yearMax]) => { - const inputs = Array.from(document.querySelectorAll('input')); - const yearInputs = inputs.filter(inp => { - const v = parseInt(inp.value, 10); - return !isNaN(v) && v >= 1900 && v <= 2100; - }); - if (yearInputs.length < 2) return false; - yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value)); - const setVal = (el, val) => { - const setter = Object.getOwnPropertyDescriptor( - HTMLInputElement.prototype, 'value' - ).set; - setter.call(el, String(val)); - el.dispatchEvent(new Event('input', {bubbles: true})); - el.dispatchEvent(new Event('change', {bubbles: true})); - }; - if (yearMin !== null) setVal(yearInputs[0], yearMin); - if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax); - const container = yearInputs[0].closest( - '[class*="filter"], [class*="year"], section, fieldset' - ) || yearInputs[0].parentElement.parentElement; - if (container) { - const btn = Array.from(container.querySelectorAll( - 'button, a, [role="button"], span[class*="apply"]' - )).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent)); - if (btn) { btn.click(); return true; } - } - yearInputs[yearInputs.length - 1].dispatchEvent( - new KeyboardEvent('keydown', { - key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true - }) - ); - return true; - }""", - [year_min, year_max], - ) - if success: - try: - page.wait_for_load_state("domcontentloaded", timeout=15_000) - except Exception: - pass - self._wait_for_listing_content(page) - logger.info("Applied year range filter: %s — %s", year_min, year_max) - return True - except Exception as exc: - logger.warning("Failed to apply year range filter: %s", exc) - return False - - def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: - # Считываем ссылки одним проходом по DOM. - self._accept_cookie_banner(page) - self._wait_for_listing_content(page) - try: - raw_items = page.eval_on_selector_all( - "a[href], [data-href], [href]", - """ - (nodes) => nodes.map((node) => ({ - href: - node.getAttribute('href') || - node.getAttribute('data-href') || - node.getAttribute('data-url') || - '', - title: node.getAttribute('title') || node.getAttribute('aria-label') || '', - text: (node.textContent || '').trim(), - })) - """, - ) - except Exception as exc: - logger.warning("collect_current_page failed on page %d: %s", page_number, exc) - raw_items = [] - total = min(len(raw_items), self.settings.listing.page_link_limit) - links: list[ListingVehicleLink] = [] - seen: set[str] = set() - for idx in range(total): - item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} - href = str(item.get("href") or "") - match = VEHICLE_HREF_RE.search(href) - if not match: - continue - lot_number = match.group(1) - absolute = urljoin(self.settings.home_url, match.group(0)) - if absolute in seen: - continue - seen.add(absolute) - title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" - links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) - if len(links) >= self.settings.listing.max_vehicles_per_run: - break - - # Fallback: на IAAI ссылки иногда не рендерятся как , - # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). - if not links: - try: - page.wait_for_timeout(1_500) - except Exception: - pass - try: - html = page.content() - except Exception as exc: - logger.debug("page.content() failed on page %d: %s", page_number, exc) - html = "" - - for absolute, lot_number in self._extract_vehicle_links_from_html(html): - if absolute in seen: - continue - seen.add(absolute) - links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) - if len(links) >= self.settings.listing.max_vehicles_per_run: - break - - if links: - logger.info( - "Page %d: recovered %d vehicle links from HTML fallback", - page_number, - len(links), - ) - next_page_detected = self._has_next_page(page) - return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) - - def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]: - if not html: - return [] - - # Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US" - normalized = html.replace("\\/", "/") - found: list[tuple[str, str]] = [] - seen: set[str] = set() - - for match in VEHICLE_HREF_RE.finditer(normalized): - lot_number = match.group(1) - absolute = urljoin(self.settings.home_url, match.group(0)) - if absolute in seen: - continue - seen.add(absolute) - found.append((absolute, lot_number)) - if len(found) >= self.settings.listing.page_link_limit: - break - - return found - - def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool: - # Запоминаем первую ссылку текущей страницы для определения смены контента. - old_first_href = "" - try: - first_link = page.locator(VEHICLE_LINK_SELECTOR).first - if first_link.count() > 0: - old_first_href = first_link.get_attribute("href") or "" - except Exception: - pass - - for selector in self._NEXT_PAGE_SELECTORS: - locator = page.locator(selector).first - if locator.count() == 0: - continue - try: - disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower() - aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower() - classes = (locator.get_attribute("class", timeout=1500) or "").lower() - except Exception: - continue - if disabled or aria_disabled == "true" or "disabled" in classes: - continue - try: - self.pacer.move_mouse_to(page, locator) - locator.click(timeout=8000) - except Exception: - continue - - if self._wait_for_navigation_result(page, old_first_href, expected_page_number): - self.pacer.after_page_change() - return True - - # Fallback для IAAI: пагинация часто рендерится как набор номеров страниц - # + стрелка с иконкой, без явного текста Next. - try: - clicked = bool(page.evaluate( - """ - () => { - const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); - const disabled = (el) => { - if (!el) return true; - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); - return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); - }; - - const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')) - .filter((el) => visible(el) && !disabled(el)); - - const current = controls.find((el) => { - const text = (el.textContent || '').trim(); - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); - return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); - }); - - if (current) { - const currentPage = parseInt((current.textContent || '').trim(), 10); - const nextNumber = controls.find((el) => { - const text = (el.textContent || '').trim(); - return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; - }); - if (nextNumber) { - nextNumber.click(); - return true; - } - } - - const iconNext = controls.find((el) => { - const text = (el.textContent || '').trim().toLowerCase(); - const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); - const title = (el.getAttribute('title') || '').trim().toLowerCase(); - const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); - const cls = (el.getAttribute('class') || '').trim().toLowerCase(); - const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); - return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); - }); - - if (iconNext) { - iconNext.click(); - return true; - } - - return false; - } - """ - )) - if clicked: - if self._wait_for_navigation_result(page, old_first_href, expected_page_number): - self.pacer.after_page_change() - return True - except Exception as exc: - logger.debug("Numeric/icon pagination fallback failed: %s", exc) - - # JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту. - try: - clicked = bool(page.evaluate( - """ - () => { - const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]')); - for (const el of candidates) { - const text = (el.textContent || '').trim().toLowerCase(); - const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); - const title = (el.getAttribute('title') || '').trim().toLowerCase(); - const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); - const cls = (el.getAttribute('class') || '').trim().toLowerCase(); - const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); - const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); - const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); - if (!disabled && visible && looksNext) { - el.click(); - return true; - } - } - return false; - } - """ - )) - if clicked: - if self._wait_for_navigation_result(page, old_first_href, expected_page_number): - self.pacer.after_page_change() - return True - except Exception as exc: - logger.debug("JS next-page fallback failed: %s", exc) - - logger.warning("Could not navigate to next page from %s", page.url) - return False - - def collect_listing_links( - self, - page: Page, - *, - make: str | None = None, - model: str | None = None, - known_origin_ids: set[str] | None = None, - max_duration_seconds: float | None = None, - ) -> dict[str, Any]: - self.open_cars_listing(page) - applied_filters = self.apply_filters(page, make=make, model=model) - started_at = time.perf_counter() - truncated_by_time_budget = False - pages: list[dict[str, object]] = [] - all_links: list[str] = [] - early_stopped = False - threshold = self.settings.listing.early_stop_threshold - - for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): - if max_duration_seconds is not None and max_duration_seconds > 0: - elapsed = time.perf_counter() - started_at - if elapsed >= max_duration_seconds: - truncated_by_time_budget = True - logger.warning( - "Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs", - page_number, - elapsed, - max_duration_seconds, - ) - break - page_result = self.collect_current_page(page, page_number=page_number) - pages.append({ - "page_number": page_result.page_number, - "source_url": page_result.source_url, - "links_found": len(page_result.vehicle_links), - "vehicle_links": [asdict(item) for item in page_result.vehicle_links], - "next_page_detected": page_result.next_page_detected, - }) - for item in page_result.vehicle_links: - if item.href not in all_links: - all_links.append(item.href) - if len(all_links) >= self.settings.listing.max_vehicles_per_run: - break - - # Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла. - # Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице). - if ( - known_origin_ids is not None - and threshold > 0.0 - and page_result.vehicle_links - ): - page_known = sum( - 1 for item in page_result.vehicle_links - if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids - ) - page_new = len(page_result.vehicle_links) - page_known - ratio = page_known / len(page_result.vehicle_links) - # Останавливаемся только если нет новых И порог превышен - if ratio >= threshold and page_new == 0: - logger.info( - "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", - page_number, ratio * 100, page_known, - len(page_result.vehicle_links), threshold * 100, - ) - early_stopped = True - break - elif page_new > 0 and ratio >= threshold: - logger.info( - "Page %d: %.0f%% known but %d new found — продолжаем", - page_number, ratio * 100, page_new, - ) - - if ( - len(all_links) >= self.settings.listing.max_vehicles_per_run - or self.settings.listing.collect_current_page_only - or not self.settings.listing.include_pagination - or not page_result.next_page_detected - ): - break - if not self.go_to_next_page(page): - break - - return { - "listing_url": self.settings.listing.cars_url, - "applied_filters": applied_filters, - "pages_collected": len(pages), - "vehicles_collected": len(all_links), - "vehicle_urls": all_links, - "early_stopped": early_stopped, - "truncated_by_time_budget": truncated_by_time_budget, - "pages": pages, - "strategy": { - "sequential": True, - "collect_current_page_only": self.settings.listing.collect_current_page_only, - "include_pagination": self.settings.listing.include_pagination, - "max_pages_per_run": self.settings.listing.max_pages_per_run, - "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run, - "early_stop_threshold": threshold, - }, - } - - @staticmethod - def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: - for selector in selectors: - locator = page.locator(selector).first - if locator.count() == 0: - continue - try: - locator.click() - locator.fill(value) - page.keyboard.press("Enter") - try: - page.wait_for_load_state("domcontentloaded", timeout=15000) - except Exception: - pass - try: - page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) - except Exception: - pass - return True - except Exception: - continue - return False - - @staticmethod - def _has_next_page(page: Page) -> bool: - for selector in ListingCollector._NEXT_PAGE_SELECTORS: - if page.locator(selector).count() > 0: - return True - try: - return bool(page.evaluate( - """ - () => { - const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); - const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => { - const cls = (el.getAttribute('class') || '').trim().toLowerCase(); - const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); - return !disabled && visible(el); - }); - - const hasExplicitNext = controls.some((el) => { - const text = (el.textContent || '').trim().toLowerCase(); - const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); - const title = (el.getAttribute('title') || '').trim().toLowerCase(); - const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); - const cls = (el.getAttribute('class') || '').trim().toLowerCase(); - const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); - return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text); - }); - - if (hasExplicitNext) { - return true; - } - - const current = controls.find((el) => { - const text = (el.textContent || '').trim(); - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); - return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); - }); - - if (!current) { - return false; - } - - const currentPage = parseInt((current.textContent || '').trim(), 10); - return controls.some((el) => { - const text = (el.textContent || '').trim(); - return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; - }); - } - """ - )) - except Exception: - return False - return False diff --git a/iaai_scraper/browser/network.py b/iaai_scraper/browser/network.py deleted file mode 100644 index 0674976..0000000 --- a/iaai_scraper/browser/network.py +++ /dev/null @@ -1,130 +0,0 @@ -import json -import logging -from dataclasses import dataclass, field -from typing import Any -from urllib.parse import urlparse - -from playwright.sync_api import Page, Request, Response - -from ..core.config import Settings - -logger = logging.getLogger("iaai_scraper.network") - - -@dataclass -class NetworkCapture: - # Перехватчик сетевых запросов. - - settings: Settings - requests: list[dict[str, Any]] = field(default_factory=list) - json_responses: list[dict[str, Any]] = field(default_factory=list) - _seen_req: set[str] = field(default_factory=set) - _seen_resp: set[str] = field(default_factory=set) - _origin: str | None = None - _page: Any = field(default=None) - - def attach(self, page: Page, origin_url: str | None = None) -> None: - # Снимаем старые подписки перед повторным attach. - try: - page.remove_listener("request", self._on_request) - page.remove_listener("response", self._on_response) - except Exception: - pass - # Подписка на сетевые события - try: - self._origin = urlparse(origin_url or page.url).netloc.lower() or None - except Exception: - self._origin = None - self._page = page - page.on("request", self._on_request) - page.on("response", self._on_response) - - def _is_same_origin(self, url: str) -> bool: - # Фильтр по домену - if not self.settings.capture.capture_same_origin_only or not self._origin: - return True - netloc = urlparse(url).netloc.lower() - return netloc == self._origin or netloc.endswith(".iaai.com") - - def _on_request(self, request: Request) -> None: - # Берём только xhr/fetch - if request.resource_type not in {"xhr", "fetch"}: - return - if not self._is_same_origin(request.url): - return - if len(self.requests) >= self.settings.capture.max_requests: - return - key = f"{request.method}:{request.url}:{request.post_data or ''}" - # Убираем дубли - if key in self._seen_req: - return - self._seen_req.add(key) - self.requests.append({ - "url": request.url, "method": request.method, - "resource_type": request.resource_type, "post_data": request.post_data, - }) - - def _on_response(self, response: Response) -> None: - # Сохраняем JSON - request = response.request - if request.resource_type not in {"xhr", "fetch"}: - return - if not self._is_same_origin(response.url): - return - if len(self.json_responses) >= self.settings.capture.max_json_responses: - return - if not self._is_json(response): - return - key = f"{request.method}:{response.url}:{response.status}" - if key in self._seen_resp: - return - self._seen_resp.add(key) - try: - payload = response.json() - except Exception: - try: - payload = json.loads(response.text()) - except Exception: - return - self.json_responses.append({ - "url": response.url, "status": response.status, - "request_method": request.method, "post_data": request.post_data, - "resource_type": request.resource_type, "payload": payload, - "category": self._categorize(response.url), - }) - - @staticmethod - def _is_json(response: Response) -> bool: - ct = (response.headers.get("content-type") or "").lower() - if "application/json" in ct or "+json" in ct: - return True - url = response.url.lower() - return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) - - @staticmethod - def _categorize(url: str) -> str: - # Простая категория URL - low = url.lower() - mapping = { - "images": ["image", "media", "photos", "gallery"], - "bids": ["bid", "offer", "buy-now", "buynow"], - "auction": ["auction", "sale", "lane", "branch"], - "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], - "documents": ["title", "document", "report"], - } - for cat, markers in mapping.items(): - if any(m in low for m in markers): - return cat - return "other" - - def export(self) -> dict[str, Any]: - responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) - return { - "requests": self.requests, - "json_responses": responses, - "capture_limits": { - "same_origin_only": self.settings.capture.capture_same_origin_only, - "max_requests": self.settings.capture.max_requests, - "max_json_responses": self.settings.capture.max_json_responses, - }, - } diff --git a/iaai_scraper/browser/pace.py b/iaai_scraper/browser/pace.py deleted file mode 100644 index 10c2551..0000000 --- a/iaai_scraper/browser/pace.py +++ /dev/null @@ -1,46 +0,0 @@ -import random -import time - -from playwright.sync_api import Locator, Page - -from ..core.config import Settings - - -class HumanPacer: - # Случайные паузы между действиями. - - def __init__(self, settings: Settings) -> None: - self.settings = settings - - def pause(self, min_s: float, max_s: float) -> None: - if self.settings.pace.enabled: - time.sleep(random.uniform(min_s, max_s)) - - def after_listing_open(self) -> None: - self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) - - def after_filter_action(self) -> None: - self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) - - def before_vehicle_open(self) -> None: - self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) - - def after_vehicle_open(self) -> None: - self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) - - def between_vehicles(self) -> None: - self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) - - def after_page_change(self) -> None: - self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) - - def move_mouse_to(self, page: Page, locator: Locator) -> None: - try: - box = locator.bounding_box() - except Exception: - box = None - if not box: - return - x = box["x"] + box["width"] * random.uniform(0.2, 0.8) - y = box["y"] + box["height"] * random.uniform(0.2, 0.8) - page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py deleted file mode 100644 index c3b76d5..0000000 --- a/iaai_scraper/cli.py +++ /dev/null @@ -1,82 +0,0 @@ -import argparse -from pathlib import Path - -from .core.config import Settings -from .core.utils import save_to_json -from .scraper import IAAIScraper - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="IAAI scraper CLI") - parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") - parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") - subparsers = parser.add_subparsers(dest="command", required=True) - - default_output_dir = Path("artifacts/json") - - subparsers.add_parser("init-db", help="Create DB tables") - - listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page") - listing_parser.add_argument("--make", default=None) - listing_parser.add_argument("--model", default=None) - listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json")) - - scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page") - scrape_parser.add_argument("vehicle_url") - scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json")) - - sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle") - sync_vehicle_parser.add_argument("vehicle_url") - sync_vehicle_parser.add_argument("--lane", default="iaai") - sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json")) - - sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles") - sync_listing_parser.add_argument("--make", default=None) - sync_listing_parser.add_argument("--model", default=None) - sync_listing_parser.add_argument("--lane", default="iaai_cars") - sync_listing_parser.add_argument("--limit", type=int, default=None) - sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) - sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json")) - - return parser - - -def main() -> None: - parser = build_parser() - args = parser.parse_args() - - runtime_settings: Settings | None = None - if args.headless is not None or args.debug: - runtime_settings = Settings() - if args.headless is not None: - runtime_settings.headless = args.headless == "true" - if args.debug: - runtime_settings.log_level = "DEBUG" - - with IAAIScraper(runtime_settings) as scraper: - if args.command == "init-db": - data = scraper.init_db() - print(f"DB initialized: {data}") - return - elif args.command == "collect-listing": - data = scraper.collect_listing(make=args.make, model=args.model) - elif args.command == "scrape-vehicle": - data = scraper.scrape_vehicle_detail(args.vehicle_url) - elif args.command == "sync-vehicle": - data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) - else: - only_new = None if args.only_new is None else args.only_new == "true" - data = scraper.sync_listing( - make=args.make, - model=args.model, - lane=args.lane, - limit=args.limit, - only_new=only_new, - ) - - save_to_json(data, Path(args.output)) - print(f"Saved to {Path(args.output).resolve()}") - - -if __name__ == "__main__": - main() diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py deleted file mode 100644 index e105ec1..0000000 --- a/iaai_scraper/core/config.py +++ /dev/null @@ -1,295 +0,0 @@ -import json -import os -from dataclasses import dataclass, field -from pathlib import Path - -from dotenv import load_dotenv - -load_dotenv() - - -TRUE_VALUES = {"1", "true", "yes", "on"} - - -# Хелперы для чтения env-переменных с приведением типов - -def _env_str(name: str, default: str) -> str: - value = os.getenv(name) - return value if value is not None else default - - -def _env_optional_str(name: str) -> str | None: - value = os.getenv(name) - if value is None: - return None - value = value.strip() - return value or None - - -def _env_path_str(name: str) -> str | None: - value = _env_optional_str(name) - if value is None: - return None - return str(Path(value).expanduser()) - - -def _env_bool(name: str, default: bool) -> bool: - fallback = "true" if default else "false" - return _env_str(name, fallback).strip().lower() in TRUE_VALUES - - -def _env_int(name: str, default: int) -> int: - return int(_env_str(name, str(default)).strip()) - - -def _env_float(name: str, default: float) -> float: - return float(_env_str(name, str(default)).strip()) - - -# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) - -@dataclass(slots=True) -class FingerprintConfig: - user_agent: str = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/135.0.0.0 Safari/537.36" - ) - viewport_presets: tuple[dict[str, int], ...] = ( - {"width": 1920, "height": 1080}, - {"width": 1600, "height": 900}, - {"width": 1536, "height": 864}, - {"width": 1440, "height": 900}, - {"width": 1366, "height": 768}, - ) - timezone_candidates: tuple[str, ...] = ( - "America/New_York", - "America/Chicago", - "America/Los_Angeles", - ) - locale: str = "en-US" - sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' - - -# Конфиг перехвата сетевых запросов (лимиты на кол-во) - -@dataclass(slots=True) -class CaptureConfig: - capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True) - max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40) - max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30) - - -# Конфиг пауз между действиями (имитация человека) - -@dataclass(slots=True) -class HumanPaceConfig: - enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True) - after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5) - after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2) - after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5) - after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2) - before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) - before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) - after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05) - after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15) - between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05) - between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15) - after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8) - after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8) - - -# Конфиг сбора листинга (URL, лимиты страниц и машин) - -@dataclass(slots=True) -class ListingConfig: - cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") - max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999) - max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000) - page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500) - include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True) - collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False) - # Порог раннего останова: если доля уже известных машин на странице >= этого значения, - # прекращаем листать — все новые машины уже найдены. 0 = отключено. - early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8) - # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин). - # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка. - # Пустая строка = без сегментации (backward compatible). - listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "") - - -# Список брендов IAAI для автоматической сегментации. -# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким. -IAAI_DEFAULT_MAKES: tuple[str, ...] = ( - "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", - "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU", - "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI", - "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN", - "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE", - "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI", - "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI", - "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART", - "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI", - "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS", - "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE", -) - -# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов. -IAAI_PAGINATION_CEILING = 22_600 - -# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам. -_LARGE_MAKES: frozenset[str] = frozenset({ - "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", - "KIA", "DODGE", "JEEP", -}) -_YEAR_SPLITS: tuple[tuple[int, int], ...] = ( - (1900, 2012), - (2013, 2019), - (2020, 2027), -) - - -def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]: - """Парсит IAAI_LISTING_SEGMENTS в список сегментов. - - Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None). - Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES. - Крупные бренды автоматически разбиваются по диапазонам годов. - """ - raw = raw.strip() - if not raw: - return [] - if raw.lower() == "auto": - segments: list[dict[str, str | int | None]] = [] - for m in IAAI_DEFAULT_MAKES: - if m in _LARGE_MAKES: - for yr_min, yr_max in _YEAR_SPLITS: - segments.append({"make": m, "year_min": yr_min, "year_max": yr_max}) - else: - segments.append({"make": m, "year_min": None, "year_max": None}) - return segments - try: - data = json.loads(raw) - except (json.JSONDecodeError, ValueError): - return [] - if not isinstance(data, list): - return [] - segments = [] - for item in data: - if isinstance(item, str): - segments.append({"make": item.upper(), "year_min": None, "year_max": None}) - elif isinstance(item, dict): - segments.append({ - "make": str(item.get("make") or "").upper() or None, - "year_min": int(item["year_min"]) if item.get("year_min") is not None else None, - "year_max": int(item["year_max"]) if item.get("year_max") is not None else None, - }) - return segments - - -# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) - -@dataclass(slots=True) -class DatabaseConfig: - url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper") - echo: bool = _env_bool("IAAI_DATABASE_ECHO", False) - pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5) - max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10) - pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800) - auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False) - - -# --- Конфиг Redis (URL для Celery broker) --- - -@dataclass(slots=True) -class RedisConfig: - url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0") - socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) - socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) - health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) - - -# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- - -@dataclass(slots=True) -class CeleryConfig: - broker_url: str = _env_str("CELERY_BROKER_URL", "") - result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") - task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) - task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) - worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) - worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) - broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) - beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) - beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None - batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) - parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8) - block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True) - - -# --- Конфиг прокси (server, username, password) --- - -@dataclass(slots=True) -class ProxyConfig: - server: str | None = _env_optional_str("IAAI_PROXY_SERVER") - username: str | None = _env_optional_str("IAAI_PROXY_USERNAME") - password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD") - - @property - def enabled(self) -> bool: - return bool(self.server) - - def to_playwright_dict(self) -> dict[str, str] | None: - if not self.server: - return None - result: dict[str, str] = {"server": self.server} - if self.username: - result["username"] = self.username - if self.password: - result["password"] = self.password - return result - - -# Главный объект настроек: собирает все блоки конфигурации - -@dataclass(slots=True) -class Settings: - home_url: str = "https://www.iaai.com/" - default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000) - network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400) - fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000) - fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1) - fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000) - max_retries: int = _env_int("IAAI_MAX_RETRIES", 3) - retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5) - retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0) - retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25) - headless: bool = _env_bool("IAAI_HEADLESS", True) - browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto") - log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO") - log_file: str | None = _env_optional_str("IAAI_LOG_FILE") - enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True) - sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False) - raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON") - tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE") - runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE") - scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60) - fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) - capture: CaptureConfig = field(default_factory=CaptureConfig) - pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) - listing: ListingConfig = field(default_factory=ListingConfig) - database: DatabaseConfig = field(default_factory=DatabaseConfig) - redis: RedisConfig = field(default_factory=RedisConfig) - celery: CeleryConfig = field(default_factory=CeleryConfig) - proxy: ProxyConfig = field(default_factory=ProxyConfig) - - @property - def parallel_tabs(self) -> int: - return self.celery.parallel_tabs - - @property - def block_resources(self) -> bool: - return self.celery.block_resources - -# Глобальный синглтон — используется по умолчанию во всех модулях. -settings = Settings() diff --git a/iaai_scraper/core/exceptions.py b/iaai_scraper/core/exceptions.py deleted file mode 100644 index d3b2872..0000000 --- a/iaai_scraper/core/exceptions.py +++ /dev/null @@ -1,14 +0,0 @@ -class ScraperError(Exception): - """Базовое исключение скрапера.""" - - -class AntiBotDetectedError(ScraperError): - """Вызывается, когда сайт блокирует автоматизацию.""" - - -class SiteStructureChangedError(ScraperError): - """Вызывается, когда структура страницы изменилась и данных не хватает.""" - - -class ListingResumeError(ScraperError): - """Вызывается, когда resume по checkpoint больше недостижим.""" diff --git a/iaai_scraper/core/retry.py b/iaai_scraper/core/retry.py deleted file mode 100644 index 2fce32c..0000000 --- a/iaai_scraper/core/retry.py +++ /dev/null @@ -1,53 +0,0 @@ -import logging -import random -import time -from collections.abc import Callable -from functools import wraps -from typing import Any - -from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError - -from .exceptions import AntiBotDetectedError - -logger = logging.getLogger("iaai_scraper.retry") - -# Типы исключений, при которых retry имеет смысл. -RETRYABLE_EXCEPTIONS = ( - PlaywrightTimeoutError, - Error, - ConnectionError, - OSError, - TimeoutError, - AntiBotDetectedError, -) - - -def retryable( - max_attempts: int, - delay_seconds: float = 2.5, - backoff_multiplier: float = 2.0, - jitter_seconds: float = 0.0, -) -> Callable[[Callable[..., Any]], Callable[..., Any]]: - def decorator(func: Callable[..., Any]) -> Callable[..., Any]: - @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: - last_error: Exception | None = None - for attempt in range(1, max_attempts + 1): - try: - return func(*args, **kwargs) - except RETRYABLE_EXCEPTIONS as exc: - last_error = exc - logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) - if attempt < max_attempts: - sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1)) - if jitter_seconds > 0: - sleep_for += random.uniform(0, jitter_seconds) - logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for) - time.sleep(sleep_for) - if last_error is not None: - raise last_error - raise RuntimeError("Retry wrapper failed without a captured exception") - - return wrapper - - return decorator diff --git a/iaai_scraper/core/runtime_config.py b/iaai_scraper/core/runtime_config.py deleted file mode 100644 index 205a154..0000000 --- a/iaai_scraper/core/runtime_config.py +++ /dev/null @@ -1,301 +0,0 @@ -import json -import logging -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - - -logger = logging.getLogger("iaai_scraper.runtime_config") - - -def _normalize_text(value: str) -> str: - return value.strip().casefold() - - -def _text_tuple(values: Any) -> tuple[str, ...]: - return tuple( - str(item).strip() - for item in (values or []) - if str(item).strip() - ) - - -def _int_tuple(values: Any) -> tuple[int, ...]: - result: list[int] = [] - for value in values or []: - try: - result.append(int(value)) - except (TypeError, ValueError): - continue - return tuple(result) - - -def _optional_int(value: Any) -> int | None: - if value in (None, ""): - return None - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _optional_bool(value: Any) -> bool | None: - if value is None: - return None - if isinstance(value, bool): - return value - if isinstance(value, str): - normalized = value.strip().casefold() - if normalized in {"1", "true", "yes", "on"}: - return True - if normalized in {"0", "false", "no", "off"}: - return False - return None - - -@dataclass(slots=True) -class RuntimeSyncConfig: - name: str | None = None - ids_initial_size: int | None = None - ids_next_size: int | None = None - ids_max_pages: int | None = None - condition_check_enabled: bool | None = None - lane: str | None = None - only_new: bool | None = None - limit: int | None = None - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig": - data = data or {} - name = str(data.get("name")).strip() if data.get("name") else None - lane = str(data.get("lane")).strip() if data.get("lane") else None - return cls( - name=name or None, - ids_initial_size=_optional_int(data.get("ids_initial_size")), - ids_next_size=_optional_int(data.get("ids_next_size")), - ids_max_pages=_optional_int(data.get("ids_max_pages")), - condition_check_enabled=_optional_bool(data.get("condition_check_enabled")), - lane=lane or None, - only_new=_optional_bool(data.get("only_new")), - limit=_optional_int(data.get("limit")), - ) - - -@dataclass(slots=True) -class RuntimeListingConfig: - make: str | None = None - model: str | None = None - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig": - data = data or {} - make = str(data.get("make")).strip() if data.get("make") else None - model = str(data.get("model")).strip() if data.get("model") else None - return cls(make=make or None, model=model or None) - - -@dataclass(slots=True) -class RuntimeFieldFilters: - brands: tuple[str, ...] = () - models: tuple[str, ...] = () - years: tuple[int, ...] = () - body_types: tuple[str, ...] = () - colors: tuple[str, ...] = () - drives: tuple[str, ...] = () - gearboxes: tuple[str, ...] = () - locations: tuple[str, ...] = () - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters": - data = data or {} - return cls( - brands=_text_tuple(data.get("brands")), - models=_text_tuple(data.get("models")), - years=_int_tuple(data.get("years")), - body_types=_text_tuple(data.get("body_types")), - colors=_text_tuple(data.get("colors")), - drives=_text_tuple(data.get("drives")), - gearboxes=_text_tuple(data.get("gearboxes")), - locations=_text_tuple(data.get("locations")), - ) - - def is_empty(self) -> bool: - return not any([ - self.brands, - self.models, - self.years, - self.body_types, - self.colors, - self.drives, - self.gearboxes, - self.locations, - ]) - - def matches(self, values: dict[str, Any]) -> bool: - return all([ - self._match_text(self.brands, values.get("brand")), - self._match_text(self.models, values.get("model")), - self._match_int(self.years, values.get("year")), - self._match_text(self.body_types, values.get("body_type")), - self._match_text(self.colors, values.get("color")), - self._match_text(self.drives, values.get("drive")), - self._match_text(self.gearboxes, values.get("gearbox")), - self._match_text(self.locations, values.get("location")), - ]) - - @staticmethod - def _match_text(allowed: tuple[str, ...], value: Any) -> bool: - if not allowed: - return True - normalized = _normalize_text(str(value or "")) - return normalized in {_normalize_text(item) for item in allowed} - - @staticmethod - def _match_int(allowed: tuple[int, ...], value: Any) -> bool: - if not allowed: - return True - parsed = _optional_int(value) - return parsed in set(allowed) - - -@dataclass(slots=True) -class RuntimeRangeFilter: - min: int | None = None - max: int | None = None - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter": - data = data or {} - return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max"))) - - def is_empty(self) -> bool: - return self.min is None and self.max is None - - def matches(self, value: Any) -> bool: - parsed = _optional_int(value) - if parsed is None: - return self.is_empty() - if self.min is not None and parsed < self.min: - return False - if self.max is not None and parsed > self.max: - return False - return True - - -@dataclass(slots=True) -class RuntimeFlagFilters: - damaged_only: bool | None = None - run_and_drive: bool | None = None - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters": - data = data or {} - return cls( - damaged_only=_optional_bool(data.get("damaged_only")), - run_and_drive=_optional_bool(data.get("run_and_drive")), - ) - - def is_empty(self) -> bool: - return self.damaged_only is None and self.run_and_drive is None - - def matches(self, values: dict[str, Any]) -> bool: - if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only: - return False - if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive: - return False - return True - - -@dataclass(slots=True) -class RuntimeFiltersConfig: - include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) - exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) - price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) - mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) - flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters) - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig": - data = data or {} - legacy_fields = RuntimeFieldFilters.from_dict(data) - include_payload = data.get("include") - exclude_payload = data.get("exclude") - - flat_exclude_payload = { - "brands": data.get("exclude_brands"), - "models": data.get("exclude_models"), - "years": data.get("exclude_years"), - "body_types": data.get("exclude_body_types"), - "colors": data.get("exclude_colors"), - "drives": data.get("exclude_drives"), - "gearboxes": data.get("exclude_gearboxes"), - "locations": data.get("exclude_locations"), - } - - include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields - exclude = ( - RuntimeFieldFilters.from_dict(exclude_payload) - if exclude_payload is not None - else RuntimeFieldFilters.from_dict(flat_exclude_payload) - ) - - return cls( - include=include, - exclude=exclude, - price=RuntimeRangeFilter.from_dict(data.get("price")), - mileage=RuntimeRangeFilter.from_dict(data.get("mileage")), - flags=RuntimeFlagFilters.from_dict(data.get("flags")), - ) - - def is_empty(self) -> bool: - return all([ - self.include.is_empty(), - self.exclude.is_empty(), - self.price.is_empty(), - self.mileage.is_empty(), - self.flags.is_empty(), - ]) - - def matches(self, values: dict[str, Any]) -> bool: - if not self.include.matches(values): - return False - if not self._matches_exclude(values): - return False - if not self.price.matches(values.get("price")): - return False - if not self.mileage.matches(values.get("mileage")): - return False - if not self.flags.matches(values): - return False - return True - - def _matches_exclude(self, values: dict[str, Any]) -> bool: - if self.exclude.is_empty(): - return True - return not self.exclude.matches(values) - - -@dataclass(slots=True) -class RuntimeConfig: - sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig) - listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig) - filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig) - - @classmethod - def from_file(cls, config_path: str | None) -> "RuntimeConfig": - if not config_path: - return cls() - path = Path(config_path) - if not path.exists(): - logger.info("Runtime config file not found: %s", path) - return cls() - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except Exception as exc: - logger.warning("Failed to read runtime config %s: %s", path, exc) - return cls() - return cls( - sync=RuntimeSyncConfig.from_dict(payload.get("sync")), - listing=RuntimeListingConfig.from_dict(payload.get("listing")), - filters=RuntimeFiltersConfig.from_dict(payload.get("filters")), - ) diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py deleted file mode 100644 index 7998e6a..0000000 --- a/iaai_scraper/parsing/mapper.py +++ /dev/null @@ -1,405 +0,0 @@ -import hashlib -import re -from datetime import datetime, timezone -from string import ascii_letters, digits -from typing import Any -from urllib.parse import urlparse - -from ..core.utils import first_non_empty -from ..storage.enums import ( - BODY_TYPE_ENUM_VALUES, - COUNTRY_ENUM_VALUES, - CURRENCY_ENUM_VALUES, - DRIVE_ENUM_VALUES, - GEARBOX_ENUM_VALUES, - ORIGIN_ENUM_VALUES, - SELLING_TYPE_ENUM_VALUES, - STEERING_WHEEL_ENUM_VALUES, -) -from ..storage.schemas import CarRecord, ImageRecord - - -class CarMapper: - # Преобразование данных IAAI в CarRecord. - - BODY_MAP = { - "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", - "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", - "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", - "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", - "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", - "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", - } - DRIVE_MAP = { - "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", - "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", - "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", - } - GEARBOX_MAP = { - "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", - "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", - "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", - } - STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} - COUNTRY_MAP = { - "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", - "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", - } - NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} - - def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: - # Собираем нормализованную DB-модель. - vehicle_summary = vehicle_summary or {} - payload_insights = payload_insights or {} - core = payload_insights.get("vehicle_core", {}) - pricing = payload_insights.get("pricing", {}) - damage = payload_insights.get("damage", {}) - auction = payload_insights.get("auction", {}) - images = payload_insights.get("images", {}) - - origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) - parser_id = self._generate_parser_id(origin_id) - brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" - model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" - year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")])) - price = self._first_parsed_int( - [ - pricing.get("buy_now"), - pricing.get("current_bid"), - vehicle_summary.get("buy_now"), - vehicle_summary.get("current_bid"), - pricing.get("actual_cash_value"), - ], - self._to_money_int, - ) - mileage = self._parse_odometer( - first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")]) - ) - color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) - drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) - # Пытаемся определить привод из строки двигателя. - if not drive or drive == "NA": - engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")])) - if engine_text: - inferred_drive = self._normalize_drive(engine_text) - if inferred_drive and inferred_drive != "NA": - drive = inferred_drive - gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) - steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" - body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) - engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")])) - title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) - seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) - location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""])) - country = self._normalize_country(first_non_empty([core.get("country"), location, "US"])) - is_damaged = self._bool_damage(damage, vehicle_summary) - is_sold = self._bool_sold(auction) - one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) - new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) - rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) - repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) - non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) - evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None - currency = self._normalize_currency( - first_non_empty( - [ - pricing.get("currency"), - vehicle_summary.get("currency"), - pricing.get("buy_now"), - pricing.get("current_bid"), - vehicle_summary.get("buy_now"), - vehicle_summary.get("current_bid"), - "USD", - ] - ) - ) - slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))) - images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) - origin = "IAAI" - - return CarRecord( - parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, - mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, - steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, - selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car, - is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, - evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, - slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records, - ) - - @staticmethod - def _as_str(value: Any) -> str: - return "" if value is None else str(value).strip() - - @staticmethod - def _to_int(value: Any) -> int | None: - if value is None: - return None - if isinstance(value, bool): - return int(value) - if isinstance(value, (int, float)): - return int(value) - digits = re.sub(r"[^\d]", "", str(value)) - return int(digits) if digits else None - - @classmethod - def _to_money_int(cls, value: Any) -> int | None: - # Нормализация стоимости из разных форматов. - if value is None: - return None - if isinstance(value, bool): - return None - if isinstance(value, (int, float)): - return int(value) - - text = str(value).strip() - if not text: - return None - - lowered = text.lower() - if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]): - return None - - numbers = re.findall(r"\d[\d\s.,]*", text) - if not numbers: - return None - - best: int | None = None - for number in numbers: - clean = number.replace(" ", "") - if "," in clean and "." in clean: - # Поддержка 1,234.56 и 1.234,56. - if clean.rfind(",") > clean.rfind("."): - clean = clean.replace(".", "").replace(",", ".") - else: - clean = clean.replace(",", "") - elif "," in clean: - parts = clean.split(",") - # 123,45 -> 123.45, иначе разделитель тысяч. - if len(parts[-1]) in {1, 2} and len(parts) == 2: - clean = clean.replace(",", ".") - else: - clean = clean.replace(",", "") - elif "." in clean: - parts = clean.split(".") - if not (len(parts[-1]) in {1, 2} and len(parts) == 2): - clean = clean.replace(".", "") - - try: - parsed = int(float(clean)) - except ValueError: - continue - - if parsed > 0 and (best is None or parsed > best): - best = parsed - - return best - - @staticmethod - def _first_parsed_int(values: list[Any], parser) -> int | None: - for value in values: - parsed = parser(value) - if parsed is not None: - return parsed - return None - - @staticmethod - def _to_year(value: Any) -> int | None: - parsed = CarMapper._to_int(value) - if parsed is None: - return None - if 1900 <= parsed <= 2100: - return parsed - return None - - @staticmethod - def _parse_odometer(value: Any) -> int: - # Разбор пробега из строк IAAI. - if value is None: - return 0 - text = str(value).strip() - if not text: - return 0 - lowered = text.lower() - if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]): - return 0 - # Извлекаем число из строкового формата одометра. - numbers = re.findall(r"[\d,]+", text) - for num_str in numbers: - clean = num_str.replace(",", "") - if clean.isdigit() and int(clean) > 0: - return int(clean) - return 0 - - def _to_engine_cc(self, value: Any) -> int | None: - # Поддержка литров и cc. - text = str(value).lower().strip() if value is not None else "" - if not text: - return None - if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): - return int(float(liters_match.group(1)) * 1000) - if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text): - return int(cubic_match.group(1)) - return int(text) if re.match(r"^\d+$", text) else None - - def _normalize_currency(self, value: Any) -> str: - text = self._as_str(value) - upper = text.upper() or "USD" - if any(token in text for token in ["€", "EUR"]): - return "EUR" - if any(token in text for token in ["¥", "JPY"]): - return "JPY" - if any(token in text for token in ["₩", "KRW"]): - return "KRW" - if any(token in text for token in ["£", "GBP"]): - return "GBP" - if any(token in text for token in ["₽", "RUB"]): - return "RUB" - if any(token in text for token in ["AED", "د.إ"]): - return "AED" - if any(token in text for token in ["CA$", "CAD"]): - return "CAD" - - text = upper - if text in CURRENCY_ENUM_VALUES: - return text - return "USD" if "$" in str(value) else "USD" - - def _normalize_drive(self, value: Any) -> str | None: - return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") - - def _normalize_gearbox(self, value: Any) -> str | None: - return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") - - def _normalize_steering(self, value: Any) -> str | None: - return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) - - def _normalize_body_type(self, value: Any) -> str: - return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" - - def _normalize_country(self, value: Any) -> str: - fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA" - return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback - - def _normalize_selling_type(self, value: Any) -> str: - text = self._as_str(value) or "AUCTION" - return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" - - def _map_value( - self, - value: Any, - mapping: dict[str, str], - allowed_values: tuple[str, ...], - *, - empty_default: str | None, - fallback: str | None, - ) -> str | None: - # Общий helper для enum-нормализации. - text = self._as_str(value).lower() - if not text: - return empty_default - if (mapped := mapping.get(text)) and mapped in allowed_values: - return mapped - for marker, mapped in mapping.items(): - if marker in text and mapped in allowed_values: - return mapped - return fallback - - @staticmethod - def _normalize_color(value: Any) -> str: - text = str(value).strip() if value is not None else "other" - if not text: - return "other" - if "/" in text: - text = text.split("/")[0].strip() - return text.lower() or "other" - - @staticmethod - def _boolish(value: Any) -> bool: - return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} - - def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: - for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: - text = self._as_str(value).lower() - if text and text not in self.NO_DAMAGE_MARKERS: - return True - return False - - def _bool_sold(self, auction: dict[str, Any]) -> bool: - return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) - - def _build_images(self, urls: list[Any]) -> list[ImageRecord]: - # Для imageKeys берем самый большой размер. - best_by_key: dict[str, str] = {} - key_order: list[str] = [] - non_keyed: list[str] = [] - for item in urls: - if not isinstance(item, str): - continue - url = item.strip() - if not url: - continue - if m := re.search(r'imageKeys=([^&]+)', url): - img_key = m.group(1) - w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 - existing = best_by_key.get(img_key) - if existing is None: - best_by_key[img_key] = url - key_order.append(img_key) - else: - existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 - if w > existing_w: - best_by_key[img_key] = url - elif url not in non_keyed: - non_keyed.append(url) - deduped = [best_by_key[k] for k in key_order] + non_keyed - return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] - - @staticmethod - def _make_fullres_url(url: str) -> str: - if "vis.iaai.com" in url: - return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) - return url - - @staticmethod - def _make_preview_url(url: str) -> str: - if "vis.iaai.com" in url: - preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) - if preview != url: - return preview - return url - - # Формирование parser_id. - - _PARSER_ID_ALPHABET = ascii_letters + digits - - @classmethod - def _generate_parser_id(cls, origin_id: str) -> str: - # Стабильный parser_id по origin_id. - digest = hashlib.sha256(origin_id.encode()).digest() - alphabet = cls._PARSER_ID_ALPHABET - base = len(alphabet) - num = int.from_bytes(digest[:17], "big") # 17 байт = 136 бит, хватает на 22 символа - chars: list[str] = [] - for _ in range(22): - num, idx = divmod(num, base) - chars.append(alphabet[idx]) - return "car-" + "".join(chars) - - def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: - # Формат: iaai:{lot_number} (аналог copart:94323985). - for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]: - text = self._as_str(value) - if text: - return f"iaai:{text}" - tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] - if "~" in tail: - tail = tail.split("~")[0] - raw = tail or self._slugify(vehicle_url) - return f"iaai:{raw}" - - @staticmethod - def _slugify(value: str) -> str: - return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" - - diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py deleted file mode 100644 index 5a3c244..0000000 --- a/iaai_scraper/parsing/parser.py +++ /dev/null @@ -1,408 +0,0 @@ -import html as html_module -import json -import logging -import re -from typing import Any - -from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty - -logger = logging.getLogger("iaai_scraper.parsers") - - -class VehicleParser: - # Парсер данных страницы автомобиля. - - # Регулярные выражения для парсинга и детекции защиты. - _BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE) - _CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]) - _ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"]) - _CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE) - _ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE) - - SUMMARY_KEY_MAP = { - "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, - "year": {"year"}, - "make": {"make", "manufacturer", "brand"}, - "model": {"model"}, - "trim": {"trim", "series"}, - "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, - "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, - "secondary_damage": {"secondarydamage"}, - "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, - "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, - "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, - "actual_cash_value": {"actualcashvalue", "acv"}, - "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, - "keys": {"keys", "keystatus"}, - "title": {"titletype", "title", "documenttype"}, - "seller": {"seller", "sellername"}, - "location": {"location", "branchname", "auctionlocation", "branch"}, - "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, - "body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"}, - "drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"}, - "gearbox": {"transmission", "gearbox", "transmissiontype"}, - "engine": {"engine", "enginevolume", "enginetype", "enginedescription"}, - "fuel_type": {"fueltype", "fuel"}, - "cylinders": {"cylinders", "cylindercount"}, - "color": {"color", "primarycolor", "exteriorcolor"}, - } - - DOM_LABEL_MAP: dict[str, str] = { - "stock #": "lot_number", - "stock": "lot_number", - "primary damage": "primary_damage", - "secondary damage": "secondary_damage", - "odometer": "odometer", - "odometer (miles)": "odometer", - "mileage": "odometer", - "body style": "body_type", - "body type": "body_type", - "vehicle type": "body_type", - "engine": "engine", - "engine type": "engine", - "transmission": "gearbox", - "drive line type": "drive", - "driveline type": "drive", - "drive line": "drive", - "driveline": "drive", - "drive type": "drive", - "fuel type": "fuel_type", - "fuel": "fuel_type", - "cylinders": "cylinders", - "exterior/interior": "color", - "exterior color": "color", - "color": "color", - "model": "model", - "series": "trim", - "selling branch": "location", - "vehicle location": "vehicle_location", - "auction date and time": "auction_date", - "sale date": "auction_date", - "lane/run #": "lane", - "actual cash value": "actual_cash_value", - "estimated repair cost": "estimated_repair_cost", - "seller": "seller", - "title/sale doc": "title", - "title/sale doc brand": "title_brand", - "start code": "run_and_drive", - "key": "keys", - "keys": "keys", - "manufactured in": "manufactured_in", - "vehicle class": "vehicle_class", - } - - def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: - result: dict[str, str] = {} - if not dom_text: - return result - lines = [line.strip() for line in dom_text.split("\n") if line.strip()] - known_labels = set(self.DOM_LABEL_MAP.keys()) - skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"} - max_fields = len(set(self.DOM_LABEL_MAP.values())) - - for i, line in enumerate(lines): - # Ранний выход, когда уже нашли все поля. - if len(result) >= max_fields: - break - - # Сценарий 1: "метка: значение" в одной строке. - colon_pos = line.find(":") - if colon_pos > 0: - label_part = line[:colon_pos].strip().lower() - value_part = line[colon_pos + 1:].strip() - if label_part in known_labels and value_part and value_part.lower() not in skip_values: - field_name = self.DOM_LABEL_MAP[label_part] - if field_name not in result or not result[field_name]: - result[field_name] = value_part - continue - - # Сценарий 2: метка и значение на соседних строках. - clean = line.rstrip(":").strip().lower() - clean_alt = clean.rstrip("#").strip() - matched_label = None - if clean in known_labels: - matched_label = clean - elif clean_alt in known_labels: - matched_label = clean_alt - - if matched_label and i + 1 < len(lines): - value = lines[i + 1].strip() - if value.rstrip(":").lower().strip() in known_labels: - continue - if value.lower() in skip_values: - continue - field_name = self.DOM_LABEL_MAP[matched_label] - if field_name not in result or not result[field_name]: - result[field_name] = value - return result - - def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: - result: dict[str, str | None] = {"year": None, "make": None, "model": None} - title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") - if title_match: - result["year"] = title_match.group(1) - result["make"] = title_match.group(2) - result["model"] = title_match.group(3) - return result - dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") - if dom_match: - result["year"] = dom_match.group(1) - result["make"] = dom_match.group(2) - result["model"] = dom_match.group(3).strip() - return result - - def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: - page_html = page_html or "" - dom_text = dom_text or "" - network_dump = network_dump or {} - responses = network_dump.get("json_responses", []) - payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] - dom_kv = self._parse_dom_key_value_pairs(dom_text) - page_title = "" - title_match = re.search(r"]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) - if title_match: - page_title = title_match.group(1).strip() - title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) - - # Один проход по payload для всех полей. - all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP) - - summary: dict[str, Any] = {"source_url": vehicle_url} - for field, values in all_found.items(): - if field in dom_kv: - values.append(dom_kv[field]) - summary[field] = first_non_empty(values) - - summary["year"] = summary.get("year") or title_parsed.get("year") - summary["make"] = summary.get("make") or title_parsed.get("make") - summary["model"] = summary.get("model") or title_parsed.get("model") - summary["trim"] = summary.get("trim") or dom_kv.get("trim") - summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) - summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) - for dom_field, dom_value in dom_kv.items(): - if dom_field not in summary or not summary[dom_field]: - summary[dom_field] = dom_value - prices = self._extract_prices_from_text(dom_text) - if not summary.get("actual_cash_value") and prices: - summary["actual_cash_value"] = prices[0] - if not summary.get("buy_now"): - buy_now_match = self._BUY_NOW_RE.search(dom_text or "") - if buy_now_match: - summary["buy_now"] = buy_now_match.group(1) - elif prices: - summary["buy_now"] = prices[0] - if not summary.get("current_bid") and len(prices) > 1: - summary["current_bid"] = prices[1] - - embedded = self._extract_embedded_json(page_html) - for item in embedded: - p = item.get("payload") - if isinstance(p, (dict, list)): - payloads.append(p) - # Дополнительный проход по встроенному JSON. - extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP) - for field, vals in extra.items(): - if not summary.get(field): - v = first_non_empty(vals) - if v: - summary[field] = v - - # Передаём image_urls без повторного извлечения. - image_urls = summary.get("image_urls") or [] - return { - "vehicle_summary": summary, - "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls), - "embedded_json": embedded, - "dom_hints": self._dom_hints(dom_text), - "access_notes": self._build_access_notes(summary, responses), - } - - def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]: - if image_urls is None: - image_urls = self._extract_image_urls(payloads, "", vehicle_url) - return { - "vehicle_core": { - "lot_number": summary.get("lot_number"), "year": summary.get("year"), - "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), - "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), - "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), - "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), - "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), - "color": summary.get("color"), "keys": summary.get("keys"), - }, - "pricing": { - "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), - "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), - "currency": self._guess_currency(summary), - }, - "bids": self._build_bid_insights(summary, payloads), - "damage": { - "primary": summary.get("primary_damage"), - "secondary": summary.get("secondary_damage"), - "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), - }, - "auction": { - "auction_date": summary.get("auction_date"), - "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), - "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), - "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), - "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), - }, - "images": {"count": len(image_urls), "urls": image_urls}, - "source_endpoints": self._build_source_endpoints(responses), - } - - def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: - return { - "amount": summary.get("current_bid"), - "currency": self._guess_currency(summary), - "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), - "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), - } - - def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: - mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} - for item in responses: - url = item.get("url", "") - category = item.get("category", "other") - if category == "vehicle": - mapping["vehicle"].append(url) - lowered = url.lower() - if any(token in lowered for token in ["bid", "offer"]): - mapping["bids"].append(url) - if any(token in lowered for token in ["damage", "report"]): - mapping["damage"].append(url) - if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): - mapping["auction"].append(url) - elif category in mapping: - mapping[category].append(url) - return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} - - def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: - endpoints = [item.get("url", "") for item in responses] - dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None)) - return { - "images_visible": bool(summary.get("image_urls")), - "network_json_count": len(responses), - "possible_captcha": bool(dom_hints.get("has_captcha_text")), - "possible_antibot": bool(dom_hints.get("has_antibot_text")), - "observed_endpoints": endpoints[:20], - } - - @staticmethod - def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: - lowered = {key.lower() for key in keys} - values: list[Any] = [] - for payload in payloads: - values.extend(deep_find_key(payload, lowered)) - return values - - @staticmethod - def _guess_currency(summary: dict[str, Any]) -> str: - for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: - value = str(summary.get(key) or "") - if "$" in value: - return "USD" - if "€" in value: - return "EUR" - if "¥" in value: - return "JPY" - return "USD" - - @staticmethod - def _extract_lot_number(text: str) -> str | None: - match = LOT_RE.search(text or "") - return match.group(1) if match else None - - @staticmethod - def _extract_prices_from_text(text: str) -> list[str]: - return [match.group(1) for match in PRICE_RE.finditer(text or "")] - - @staticmethod - def _extract_embedded_json(html: str) -> list[dict[str, Any]]: - scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) - extracted: list[dict[str, Any]] = [] - for script_text in scripts: - if "{" not in script_text and "[" not in script_text: - continue - # Пропускаем слишком большие минифицированные блоки. - if len(script_text) > 51_200: - continue - try: - parsed = json.loads(script_text.strip()) - except Exception: - continue - extracted.append({"type": "inline_json", "payload": parsed}) - return extracted - - @staticmethod - def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: - vehicle_key = "" - key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") - if key_match: - vehicle_key = key_match.group(1) - found: list[str] = [] - for payload in payloads: - found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) - flat: list[str] = [] - seen_flat: set[str] = set() - for item in found: - if isinstance(item, str) and item.startswith("http"): - cleaned = html_module.unescape(item) - lowered = cleaned.lower() - if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: - continue - if cleaned not in seen_flat: - seen_flat.add(cleaned) - flat.append(cleaned) - elif isinstance(item, list): - for child in item: - if isinstance(child, str) and child.startswith("http"): - cleaned = html_module.unescape(child) - lowered = cleaned.lower() - if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: - continue - if cleaned not in seen_flat: - seen_flat.add(cleaned) - flat.append(cleaned) - for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: - for match in re.finditer(pattern, html or "", re.IGNORECASE): - url = html_module.unescape(match.group(1).strip()) - if not url.startswith("http") or url in seen_flat: - continue - lowered = url.lower() - if vehicle_key and vehicle_key in url: - seen_flat.add(url) - flat.append(url) - elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]): - if vehicle_key and vehicle_key not in url: - continue - seen_flat.add(url) - flat.append(url) - if vehicle_key: - for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""): - cleaned = html_module.unescape(url) - if cleaned not in seen_flat and vehicle_key in cleaned: - seen_flat.add(cleaned) - flat.append(cleaned) - filtered: list[str] = [] - for url in flat: - lowered = url.lower() - if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): - continue - if "vis.iaai.com" in lowered and "/resizer" not in lowered: - continue - filtered.append(url) - return filtered - - @staticmethod - def _dom_hints(text: str) -> dict[str, Any]: - lowered = (text or "").lower() - return { - "has_buy_now_text": "buy now" in lowered, - "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, - "has_damage_text": "damage" in lowered, - "has_title_text": "title" in lowered, - "has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS), - "has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS), - } diff --git a/iaai_scraper/proxy_bridge.py b/iaai_scraper/proxy_bridge.py deleted file mode 100644 index 251ce58..0000000 --- a/iaai_scraper/proxy_bridge.py +++ /dev/null @@ -1,317 +0,0 @@ -from __future__ import annotations - -import logging -import os -import select -import socket -import socketserver -import struct -import threading -from urllib.parse import urlsplit - -BUFFER_SIZE = 65536 -CRLF = b"\r\n" -DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1") -DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899")) -SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "") -SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002")) -SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "") -SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "") -RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60")) -MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64")) - -logger = logging.getLogger("proxy_bridge") - - -class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - allow_reuse_address = True - daemon_threads = True - - def __init__(self, server_address, request_handler_class): - super().__init__(server_address, request_handler_class) - self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS) - - def process_request_thread(self, request, client_address): - with self._worker_semaphore: - super().process_request_thread(request, client_address) - - -def _recv_exact(sock: socket.socket, size: int) -> bytes: - data = b"" - while len(data) < size: - chunk = sock.recv(size - len(data)) - if not chunk: - raise ConnectionError("Unexpected EOF from SOCKS5 server") - data += chunk - return data - - -def _socks5_connect(host: str, port: int) -> socket.socket: - if not SOCKS5_HOST: - raise RuntimeError("SOCKS5_PROXY_HOST is not configured") - - upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30) - upstream.settimeout(30) - - methods = [0x00] - if SOCKS5_USER or SOCKS5_PASS: - methods = [0x02] - upstream.sendall(bytes([0x05, len(methods), *methods])) - version, method = _recv_exact(upstream, 2) - if version != 0x05 or method == 0xFF: - upstream.close() - raise ConnectionError("SOCKS5 authentication negotiation failed") - - if method == 0x02: - username = SOCKS5_USER.encode("utf-8") - password = SOCKS5_PASS.encode("utf-8") - if len(username) > 255 or len(password) > 255: - upstream.close() - raise ValueError("SOCKS5 username/password too long") - upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password) - auth_version, auth_status = _recv_exact(upstream, 2) - if auth_version != 0x01 or auth_status != 0x00: - upstream.close() - raise ConnectionError("SOCKS5 username/password authentication failed") - - try: - socket.inet_aton(host) - addr_type = 0x01 - addr_payload = socket.inet_aton(host) - except OSError: - host_bytes = host.encode("idna") - if len(host_bytes) > 255: - upstream.close() - raise ValueError("Target host is too long for SOCKS5 domain format") - addr_type = 0x03 - addr_payload = bytes([len(host_bytes)]) + host_bytes - - request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port) - upstream.sendall(request) - - response_head = _recv_exact(upstream, 4) - version, reply, _reserved, reply_addr_type = response_head - if version != 0x05 or reply != 0x00: - upstream.close() - raise ConnectionError(f"SOCKS5 connect failed with code {reply}") - - if reply_addr_type == 0x01: - _recv_exact(upstream, 4) - elif reply_addr_type == 0x03: - domain_len = _recv_exact(upstream, 1)[0] - _recv_exact(upstream, domain_len) - elif reply_addr_type == 0x04: - _recv_exact(upstream, 16) - _recv_exact(upstream, 2) - - upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) - return upstream - - -def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None: - sockets = [left, right] - left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) - right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) - try: - while True: - readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS) - if exceptional: - break - if not readable: - logger.debug("Relay idle timeout reached; closing sockets") - return - for current in readable: - other = right if current is left else left - data = current.recv(BUFFER_SIZE) - if not data: - return - other.sendall(data) - finally: - for sock in sockets: - try: - sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - try: - sock.close() - except OSError: - pass - - -class ProxyHandler(socketserver.StreamRequestHandler): - def handle(self) -> None: - try: - request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip() - if not request_line: - return - - method, target, version = request_line.split() - headers = self._read_headers() - logger.info("%s %s", method, target) - - if method.upper() == "CONNECT": - host, port = self._parse_connect_target(target) - logger.info("CONNECT %s:%s", host, port) - upstream = _socks5_connect(host, port) - self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF) - self.wfile.flush() - _relay_bidirectional(self.connection, upstream) - return - - host, port, path = self._parse_forward_target(target, headers) - upstream = _socks5_connect(host, port) - self._send_forward_request(upstream, method, path, version, headers) - body = self._read_request_body(headers) - if body: - upstream.sendall(body) - _relay_bidirectional(self.connection, upstream) - except Exception as exc: - logger.exception("Proxy bridge request failed: %s", exc) - try: - self.wfile.write( - b"HTTP/1.1 502 Bad Gateway" + CRLF - + b"Connection: close" + CRLF - + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF - + b"Bad Gateway" - ) - self.wfile.flush() - except OSError: - pass - - def _read_headers(self) -> list[tuple[str, str]]: - headers: list[tuple[str, str]] = [] - while True: - line = self.rfile.readline(BUFFER_SIZE) - if line in {CRLF, b"\n", b""}: - break - decoded = line.decode("iso-8859-1") - if ":" not in decoded: - continue - name, value = decoded.split(":", 1) - headers.append((name.strip(), value.strip())) - return headers - - @staticmethod - def _parse_connect_target(target: str) -> tuple[str, int]: - if target.startswith("["): - end = target.find("]") - if end == -1 or len(target) <= end + 2 or target[end + 1] != ":": - raise ValueError("Invalid CONNECT target") - host = target[1:end] - port_text = target[end + 2 :] - return host, int(port_text) - - host, port_text = target.rsplit(":", 1) - return host, int(port_text) - - @staticmethod - def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]: - if target.startswith("http://"): - parts = urlsplit(target) - port = parts.port or 80 - path = parts.path or "/" - if parts.query: - path += f"?{parts.query}" - return parts.hostname or "", port, path - - if target.startswith("https://"): - raise ValueError("HTTPS absolute-form request must use CONNECT") - - host_header = next((value for name, value in headers if name.lower() == "host"), "") - if not host_header: - raise ValueError("Missing Host header") - if ":" in host_header: - host, port_text = host_header.rsplit(":", 1) - return host, int(port_text), target - return host_header, 80, target - - def _send_forward_request( - self, - upstream: socket.socket, - method: str, - path: str, - version: str, - headers: list[tuple[str, str]], - ) -> None: - filtered_headers: list[tuple[str, str]] = [] - hop_by_hop = { - "proxy-connection", - "proxy-authorization", - "connection", - "keep-alive", - "te", - "trailer", - "transfer-encoding", - "upgrade", - } - for name, value in headers: - if name.lower() in hop_by_hop: - continue - filtered_headers.append((name, value)) - - request_head = [f"{method} {path} {version}\r\n"] - request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers) - request_head.append("\r\n") - upstream.sendall("".join(request_head).encode("iso-8859-1")) - - def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes: - transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "") - if "chunked" in transfer_encoding.lower(): - return self._read_chunked_request_body() - - content_length = next((value for name, value in headers if name.lower() == "content-length"), None) - if not content_length: - return b"" - return self.rfile.read(int(content_length)) - - def _read_chunked_request_body(self) -> bytes: - chunks: list[bytes] = [] - while True: - size_line = self.rfile.readline(BUFFER_SIZE) - if not size_line: - raise ConnectionError("Unexpected EOF in chunked request") - size_text = size_line.strip().split(b";", 1)[0] - chunk_size = int(size_text, 16) - chunks.append(size_line) - if chunk_size == 0: - while True: - trailer_line = self.rfile.readline(BUFFER_SIZE) - if not trailer_line: - raise ConnectionError("Unexpected EOF in chunked trailers") - chunks.append(trailer_line) - if trailer_line in {CRLF, b"\n"}: - return b"".join(chunks) - - chunk_data = self.rfile.read(chunk_size) - if len(chunk_data) != chunk_size: - raise ConnectionError("Unexpected EOF in chunk body") - chunks.append(chunk_data) - chunk_end = self.rfile.read(2) - if chunk_end != CRLF: - raise ConnectionError("Invalid chunk terminator") - chunks.append(chunk_end) - - -def main() -> None: - if not SOCKS5_HOST: - raise SystemExit("SOCKS5_PROXY_HOST is required") - - logging.basicConfig( - level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(), - format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s", - ) - - with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server: - logger.info( - "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)", - DEFAULT_LISTEN_HOST, - DEFAULT_LISTEN_PORT, - SOCKS5_HOST, - SOCKS5_PORT, - MAX_WORKERS, - ) - server.serve_forever() - - -if __name__ == "__main__": - main() diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py deleted file mode 100644 index 94bdd98..0000000 --- a/iaai_scraper/scraper.py +++ /dev/null @@ -1,2101 +0,0 @@ -import json -import logging -import os -import random -import re -import signal -import time -import uuid -import html as html_module -from concurrent.futures import Future, ThreadPoolExecutor, as_completed -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Callable -from urllib.parse import urlsplit, urlunsplit -from urllib.request import Request, urlopen - -import urllib3 - -try: - import orjson as _orjson # ~3-5× быстрее stdlib json на парсинге - _HAS_ORJSON = True -except ImportError: - _orjson = None - _HAS_ORJSON = False - -from playwright.sync_api import Error as PlaywrightError -from playwright.sync_api import BrowserContext, Page, sync_playwright -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError - -from .browser import BrowserFactory, HumanPacer, NetworkCapture -from .core.config import Settings, settings, parse_listing_segments -from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError -from .core.logs import set_trace_id, setup_logging -from .core.retry import retryable -from .core.runtime_config import RuntimeConfig -from .core.utils import save_to_json -from .parsing.mapper import CarMapper -from .parsing.parser import VehicleParser -from .storage.db import PersistenceService -from .browser.listing import ListingCollector, ListingPageResult -from .storage.schemas import CarRecord - -logger = logging.getLogger("iaai_scraper.scraper") -VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) -HTML_TAG_RE = re.compile(r"<[^>]+>") -SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) - -# ── Anti-stall watchdog ── -# Жёсткие верхние границы на per-page операции браузера. Если Playwright "тихо" завис -# (частая реакция IAAI на затяжной скрапинг — page hangs без raise), SIGALRM вернёт -# управление и существующая recover-логика (_reopen_listing_and_resume) стартует -# новый context с нуля. -_PAGE_COLLECT_TIMEOUT_S = 90 -_PAGE_NEXT_TIMEOUT_S = 60 -# Проактивная ротация контекста отключена: reactive-восстановление -# (retry при пустой странице + _reopen_listing_and_resume) уже закрывает -# anti-bot соскоки, а проактивная ротация требовала долистывать кликами -# после reopen и конфликтовала с лимитом max_nav_pages в recovery-пути. -# Включить можно переменной окружения IAAI_CONTEXT_ROTATE_EVERY_PAGES. -_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("IAAI_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) - - -class PageOperationTimeoutError(Exception): - """Browser page operation exceeded per-op watchdog timeout.""" - - -class _PageOpWatchdog: - """SIGALRM-based watchdog. - - Работает только в главном потоке процесса (Celery prefork child — это ок). - В других контекстах — no-op. - """ - - def __init__(self, seconds: int, label: str) -> None: - self.seconds = max(1, int(seconds)) - self.label = label - self._old_handler = None - self._active = False - - def _handler(self, signum, frame): # noqa: ARG002 - raise PageOperationTimeoutError( - f"Page operation '{self.label}' exceeded {self.seconds}s watchdog" - ) - - def __enter__(self): - import threading as _threading - if _threading.current_thread() is not _threading.main_thread(): - return self - if not hasattr(signal, "SIGALRM"): - return self - try: - self._old_handler = signal.signal(signal.SIGALRM, self._handler) - signal.alarm(self.seconds) - self._active = True - except (ValueError, OSError): - # signal можно выставлять только из main thread; в остальном пропускаем. - self._active = False - return self - - def __exit__(self, exc_type, exc, tb): - if not self._active: - return False - try: - signal.alarm(0) - if self._old_handler is not None: - signal.signal(signal.SIGALRM, self._old_handler) - except (ValueError, OSError): - pass - return False - - -class IAAIScraper: - - @staticmethod - def _is_protection_or_network_error(exc: Exception) -> bool: - message = str(exc).lower() - signals = ( - "captcha", - "antibot", - "blocked", - "challenge", - "ns_error_net_interrupt", - "navigation", - "timeout", - "403", - "429", - ) - return any(signal in message for signal in signals) - - @staticmethod - def _html_to_text(html: str) -> str: - if not html: - return "" - cleaned = SCRIPT_STYLE_RE.sub(" ", html) - text = HTML_TAG_RE.sub(" ", cleaned) - text = html_module.unescape(text) - return re.sub(r"\s+", " ", text).strip() - - @staticmethod - def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None: - dom_hints = parsed.get("dom_hints", {}) or {} - access_notes = parsed.get("access_notes", {}) or {} - summary = parsed.get("vehicle_summary", {}) or {} - - has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model")) - - if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity: - raise AntiBotDetectedError(f"IAAI anti-bot detected for {vehicle_url}") - - if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity: - raise AntiBotDetectedError(f"IAAI blocked or challenged request for {vehicle_url}") - - if not has_identity: - raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") - - @staticmethod - def _extract_origin_id_from_url(vehicle_url: str) -> str | None: - match = VEHICLE_ID_RE.search(vehicle_url) - if not match: - return None - return match.group(1) - - @staticmethod - def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None: - raw_id = IAAIScraper._extract_origin_id_from_url(vehicle_url) - if not raw_id: - return None - return f"iaai:{raw_id}" - - @staticmethod - def _normalize_vehicle_url(vehicle_url: str) -> str: - try: - parts = urlsplit(vehicle_url) - return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) - except Exception: - return vehicle_url - - def __init__(self, runtime_settings: Settings | None = None) -> None: - self.settings = runtime_settings or settings - setup_logging(self.settings.log_level, self.settings.log_file) - self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) - self.trace_id = uuid.uuid4().hex[:12] - set_trace_id(self.trace_id) - self.playwright = None - self.browser = None - self.context: BrowserContext | None = None - self.browser_factory = BrowserFactory(self.settings) - self.pacer = HumanPacer(self.settings) - self.listing_collector = ListingCollector(self.settings, self.pacer) - self.vehicle_parser = VehicleParser() - self.car_mapper = CarMapper() - self.persistence = PersistenceService(self.settings) - self._shutdown_requested = False - # HTTP pool: при parallel_tabs=24 и concurrency=4 пик ≈ 96 конкурентных сокетов; - # даём запас до 256, чтобы не упираться в PoolError под всплесками PX/retry. - # TCP_NODELAY выключает Nagle: для коротких HTTPS-запросов с keep-alive - # это снижает tail-latency на десятки ms. - import socket as _socket - _socket_options = [ - (_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1), - (_socket.SOL_SOCKET, _socket.SO_KEEPALIVE, 1), - ] - proxy_url = self.settings.proxy.server - if proxy_url: - _proxy_kwargs: dict = { - "num_pools": 8, - "maxsize": 256, - "block": False, - "retries": False, - "timeout": urllib3.Timeout(connect=5, read=20), - "socket_options": _socket_options, - } - if self.settings.proxy.username: - _proxy_kwargs["proxy_headers"] = urllib3.make_headers( - proxy_basic_auth=f"{self.settings.proxy.username}:{self.settings.proxy.password or ''}" - ) - self._http_pool: urllib3.PoolManager = urllib3.ProxyManager(proxy_url, **_proxy_kwargs) - logger.info("HTTP fast-path using proxy: %s", proxy_url) - else: - self._http_pool = urllib3.PoolManager( - num_pools=8, - maxsize=256, - block=False, - retries=False, - timeout=urllib3.Timeout(connect=5, read=20), - socket_options=_socket_options, - ) - - def _new_trace_id(self, prefix: str) -> str: - trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" - self.trace_id = trace_id - set_trace_id(trace_id) - return trace_id - - def __enter__(self) -> "IAAIScraper": - if self.playwright is None: - self.playwright = sync_playwright().start() - if self.browser is None: - self.browser = self.browser_factory.create_browser(self.playwright) - return self - - def __exit__(self, exc_type, exc, tb) -> None: - self.close() - - def close(self) -> None: - if self.context is not None: - try: - self.context.close() - except PlaywrightError: - pass - finally: - self.context = None - if self.browser is not None: - try: - self.browser.close() - except PlaywrightError: - pass - finally: - self.browser = None - if self.playwright is not None: - try: - self.playwright.stop() - except PlaywrightError: - pass - finally: - self.playwright = None - if getattr(self, "_http_pool", None) is not None: - try: - self._http_pool.clear() - except Exception: - pass - finally: - self._http_pool = None - - def _new_context(self) -> BrowserContext: - if self.browser is None: - self.__enter__() - if self.context: - try: - self.context.close() - except PlaywrightError: - pass - self.context = self.browser_factory.create_context(self.browser) - return self.context - - def init_db(self): - self.persistence.create_tables() - return {"status": "ok", "database_url": self.settings.database.url} - - def _warmup_visit(self, page: Page) -> None: - try: - logger.info("Warmup: visiting homepage to pass anti-bot challenge...") - page.goto(self.settings.home_url, wait_until="commit", timeout=30_000) - try: - page.wait_for_load_state("domcontentloaded", timeout=6_000) - except PlaywrightTimeoutError: - pass - try: - page.wait_for_function("() => document.title && document.title.length > 3", timeout=4_000) - except PlaywrightTimeoutError: - pass - time.sleep(0.3) - logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50]) - except Exception as e: - logger.warning("Warmup visit failed: %s — continuing anyway", e) - time.sleep(1.5) - - def _get_page_with_warmup(self) -> Page: - context = self._new_context() - page = context.new_page() - self._warmup_visit(page) - return page - - def _dedupe_urls(self, raw_urls: list[str]) -> list[str]: - vehicle_urls: list[str] = [] - seen: set[str] = set() - for raw in raw_urls: - normalized = self._normalize_vehicle_url(raw) - if normalized not in seen: - seen.add(normalized) - vehicle_urls.append(normalized) - return vehicle_urls - - def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: - url_to_origin_id = { - url: self._extract_db_origin_id_from_url(url) - for url in vehicle_urls - } - candidate_origin_ids = [oid for oid in url_to_origin_id.values() if oid] - existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( - vehicle_urls, candidate_origin_ids, - ) - known_urls = { - url for url in vehicle_urls - if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) - } - skipped = len(known_urls) - if skipped: - logger.info("Filtering already known vehicles: skipped %d", skipped) - new_urls = [url for url in vehicle_urls if url not in known_urls] - return new_urls, skipped - - def _extract_page_urls( - self, - page_result: ListingPageResult, - all_raw_urls: list[str], - seen_urls: set[str], - all_listing_origin_urls: set[str] | None = None, - ) -> list[str]: - page_urls: list[str] = [] - for item in page_result.vehicle_links: - normalized = self._normalize_vehicle_url(item.href) - all_raw_urls.append(item.href) - if normalized and all_listing_origin_urls is not None: - all_listing_origin_urls.add(normalized) - if normalized and normalized not in seen_urls: - seen_urls.add(normalized) - page_urls.append(normalized) - return page_urls - - @staticmethod - def _build_segment_listing_url(base_url: str, make: str | None) -> str: - """Построить URL листинга для сегмента. Make передаётся через query-параметр.""" - if not make: - return base_url - sep = "&" if "?" in base_url else "?" - return f"{base_url}{sep}Make={make.replace(' ', '%20')}" - - def _recover_empty_listing_page( - self, - page: Page, - *, - page_number: int, - all_raw_urls: list[str], - seen_urls: set[str], - all_listing_origin_urls: set[str] | None = None, - listing_url: str | None = None, - ) -> tuple[ListingPageResult, list[str]]: - if page_number == 1: - logger.warning("Page 1 returned 0 links — retrying listing open...") - time.sleep(3) - self.listing_collector.open_cars_listing(page, url_override=listing_url) - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) - - logger.warning( - "Page %d returned 0 links — retrying current page before stopping pagination", - page_number, - ) - try: - page.reload(wait_until="domcontentloaded", timeout=30_000) - except Exception as exc: - logger.debug("Page %d reload failed during empty-page recovery: %s", page_number, exc) - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls) - - def _open_listing_page( - self, - *, - make: str | None, - model: str | None, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - ) -> tuple[Page, dict[str, str | int | None]]: - page = self._get_page_with_warmup() - try: - self.listing_collector.open_cars_listing(page, url_override=listing_url) - applied_filters = self.listing_collector.apply_filters( - page, - make=make, - model=model, - year_min=year_min, - year_max=year_max, - ) - except Exception: - page.close() - raise - return page, applied_filters - - def _reopen_listing_and_resume( - self, - *, - target_page_number: int, - make: str | None, - model: str | None, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - max_nav_pages: int = 10, - ) -> tuple[Page, dict[str, str | int | None]]: - # Ограничиваем глубину навигации: если до цели > max_nav_pages кликов — не пытаемся. - if target_page_number > max_nav_pages + 1: - raise ListingResumeError( - f"Cannot resume at page {target_page_number}: " - f"exceeds max navigation depth ({max_nav_pages} pages)" - ) - page, applied_filters = self._open_listing_page( - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) - - for expected_page in range(2, target_page_number + 1): - if not self.listing_collector.go_to_next_page(page, expected_page_number=expected_page): - page.close() - raise ListingResumeError(f"Failed to resume listing at page {target_page_number}") - - logger.warning("Listing resumed at page %d after recovery", target_page_number) - return page, applied_filters - - def _open_listing_for_stream( - self, - *, - make: str | None, - model: str | None, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - ) -> tuple[Page, dict[str, str | int | None]]: - # Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие - # сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется - # на уровне сегментов в worker/tasks.py (_save_last_completed_segment). - return self._open_listing_page( - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) - - def collect_listing( - self, - make: str | None = None, - model: str | None = None, - known_origin_ids: set[str] | None = None, - max_duration_seconds: float | None = None, - ): - page = self._get_page_with_warmup() - - try: - listing = self.listing_collector.collect_listing_links( - page, - make=make, - model=model, - known_origin_ids=known_origin_ids, - max_duration_seconds=max_duration_seconds, - ) - finally: - page.close() - - return { - "status": "ok", - **listing, - } - - def _sync_listing_streaming( - self, - *, - make: str | None, - model: str | None, - lane: str, - limit: int | None, - effective_only_new: bool, - started_at: float, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - ) -> dict[str, Any]: - batch_size = self.settings.celery.batch_size - pending_urls: list[str] = [] - seen_urls: set[str] = set() - all_raw_urls: list[str] = [] - all_listing_origin_urls: set[str] = set() - pages_info: list[dict[str, Any]] = [] - skipped_existing = 0 - total = 0 - cars_upserted = 0 - cars_failed = 0 - images_upserted = 0 - failures: list[dict[str, str]] = [] - early_stopped = False - truncated_by_time_budget = False - - known_origin_ids: set[str] | None = None - threshold = self.settings.listing.early_stop_threshold - if effective_only_new and threshold > 0.0: - try: - known_origin_ids = self.persistence.get_all_origin_ids_for_lane("iaai:") - logger.info( - "Loaded %d known origin_ids for early-stop listing", - len(known_origin_ids), - ) - except Exception as exc: - logger.warning("Could not load known origin_ids for early-stop: %s", exc) - - # Совместимость: если only_new без limit и без сегментного URL — старая схема. - # При сегментированном скрапинге (listing_url/year фильтры) всегда streaming. - if effective_only_new and (limit is None or limit <= 0) and not listing_url and year_min is None and year_max is None: - listing_payload = self.collect_listing( - make=make, - model=model, - known_origin_ids=known_origin_ids, - max_duration_seconds=None, - ) - raw_urls = list(listing_payload.get("vehicle_urls", [])) - deduped_urls = self._dedupe_urls(raw_urls) - fresh_urls, page_skipped = self._filter_known_urls(deduped_urls) - skipped_existing += page_skipped - pending_urls.extend(fresh_urls) - total = len(fresh_urls) - - for raw in raw_urls: - normalized = self._normalize_vehicle_url(raw) - if normalized: - all_listing_origin_urls.add(normalized) - - logger.info( - "Starting sync: %d vehicles to process (batch mode, only_new legacy path)", - total, - ) - - while len(pending_urls) >= batch_size: - batch_urls = pending_urls[:batch_size] - try: - batch_result = self.sync_batch(batch_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - except Exception as batch_exc: - logger.error("Legacy only_new batch failed: %s", batch_exc) - cars_failed += len(batch_urls) - failures.append({"vehicle_url": "legacy_only_new_batch", "error": str(batch_exc)}) - pending_urls = pending_urls[batch_size:] - - if pending_urls: - try: - batch_result = self.sync_batch(pending_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - except Exception as batch_exc: - logger.error("Legacy only_new final batch failed: %s", batch_exc) - cars_failed += len(pending_urls) - failures.append({"vehicle_url": "legacy_only_new_final_batch", "error": str(batch_exc)}) - - return { - "listing": listing_payload, - "total": total, - "skipped_existing": skipped_existing, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "failures": failures, - "all_listing_origin_urls": all_listing_origin_urls, - } - - # ── Pipeline-режим: батчи (HTTP + DB upsert) выполняются в фоне, - # пока главный поток продолжает собирать listing-страницы через - # Playwright. Это убирает простой Playwright во время HTTP-фазы - # (~50-65 сек на 400 машин) и даёт ~25-30% к throughput. - # max_workers=1: батчи исполняются последовательно, но ВНЕ - # главного потока, чтобы не блокировать сбор listing-а. - # Browser fallback откладывается (skip_browser_fallback=True), - # потому что Playwright не thread-safe — обработаем после loop'а. - batch_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="batch-pipeline") - pending_future: Future | None = None - pending_future_meta: tuple[int, int] | None = None # (batch_start, batch_size) - deferred_fallbacks: list[tuple[str, int]] = [] - - def _drain_pending_future() -> bool: - """Waits for in-flight batch and accumulates its result. - - Returns False if a non-recoverable error occurred and remaining - batches should be aborted (mirrors the legacy contract). - """ - nonlocal pending_future, pending_future_meta - nonlocal cars_upserted, cars_failed, images_upserted, failures - if pending_future is None: - return True - future = pending_future - meta = pending_future_meta or (0, 0) - pending_future = None - pending_future_meta = None - batch_start, batch_size_actual = meta - try: - batch_result = future.result() - except PlaywrightError as pw_exc: - logger.warning( - "Batch %d-%d browser crashed: %s. Reinitializing browser context.", - batch_start + 1, - batch_start + batch_size_actual, - pw_exc, - ) - cars_failed += batch_size_actual - failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(pw_exc)}) - try: - self._new_context() - logger.info("Browser context reinitialized successfully after crash") - return True - except Exception as reinit_exc: - logger.error("Failed to reinitialize browser: %s. Aborting remaining batches.", reinit_exc) - return False - except Exception as batch_exc: - logger.error( - "Batch %d-%d failed: %s. Continuing with next batch.", - batch_start + 1, - batch_start + batch_size_actual, - batch_exc, - ) - cars_failed += batch_size_actual - failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)}) - return True - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - deferred_fallbacks.extend(batch_result.get("deferred_fallback_urls", [])) - return True - - def _process_pending_batch(batch_urls: list[str], batch_start: int) -> bool: - nonlocal pending_future, pending_future_meta - # Drain previous batch first (if still running). - if not _drain_pending_future(): - return False - if not batch_urls: - return True - logger.info( - "Processing batch %d-%d of %d (pipelined)", - batch_start + 1, - batch_start + len(batch_urls), - total, - ) - # Cookie header обязательно вычисляем в главном потоке — Playwright - # context.cookies() использует greenlet, который нельзя дёргать из background thread'а. - cookie_header = self._cookie_header_for_context() - pending_future = batch_executor.submit( - self.sync_batch, - batch_urls, - lane=lane, - skip_browser_fallback=True, - cookie_header_override=cookie_headerkground thread'а. - cookie_header = self._cookie_header_for_context() - pending_future = batch_executor.submit( - self.sync_batch, - batch_urls, - lane=lane, - skip_browser_fallback=True, - cookie_header_override=cookie_header, - ) - pending_future_meta = (batch_start, len(batch_urls)) - return True - - page = None - try: - page, applied_filters = self._open_listing_for_stream( - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) - - pages_since_rotation = 0 - for page_number in range(1, self.settings.listing.max_pages_per_run + 1): - # Проактивная ротация контекста (опционально, по умолчанию выключена). - # Если включена — требует долистывания до page_number после reopen, - # поэтому max_nav_pages поднят под реальную глубину. - if _CONTEXT_ROTATE_EVERY_PAGES > 0 and pages_since_rotation >= _CONTEXT_ROTATE_EVERY_PAGES: - logger.warning( - "Rotating browser context at page %d (every %d pages) to reset anti-bot state", - page_number, _CONTEXT_ROTATE_EVERY_PAGES, - ) - try: - page.close() - except Exception: - pass - page = None - try: - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=max(page_number, 500), - ) - pages_since_rotation = 0 - except ListingResumeError as resume_exc: - logger.warning( - "Context rotation could not resume at page %d (%s) — stopping segment", - page_number, resume_exc, - ) - break - - try: - with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number}"): - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - except (PageOperationTimeoutError, PlaywrightError) as op_exc: - logger.warning( - "Page %d collect stalled/failed (%s) — reopening listing and resuming", - page_number, op_exc, - ) - try: - page.close() - except Exception: - pass - page = None - try: - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=max(page_number, 500), - ) - pages_since_rotation = 0 - with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number} (after reopen)"): - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - except (ListingResumeError, PageOperationTimeoutError, PlaywrightError) as resume_exc: - logger.warning( - "Cannot recover at page %d (%s) — stopping pagination for this segment", - page_number, resume_exc, - ) - break - - pages_info.append({ - "page_number": page_result.page_number, - "links_found": len(page_result.vehicle_links), - }) - pages_since_rotation += 1 - - page_urls = self._extract_page_urls( - page_result, - all_raw_urls, - seen_urls, - all_listing_origin_urls, - ) - - if not page_urls: - page_result, page_urls = self._recover_empty_listing_page( - page, - page_number=page_number, - all_raw_urls=all_raw_urls, - seen_urls=seen_urls, - all_listing_origin_urls=all_listing_origin_urls, - listing_url=listing_url, - ) - if not page_urls and page_number > 1: - logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number) - page.close() - try: - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=max(page_number, 500), - ) - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - page_urls = self._extract_page_urls( - page_result, - all_raw_urls, - seen_urls, - all_listing_origin_urls, - ) - except ListingResumeError as resume_exc: - logger.warning( - "Cannot resume at page %d (%s) — stopping pagination for this segment", - page_number, resume_exc, - ) - page = None - page_urls = [] - if not page_urls: - logger.info("Page %d: 0 new links, stopping pagination", page_number) - break - - if known_origin_ids is not None and threshold > 0.0 and page_result.vehicle_links: - page_known = sum( - 1 - for item in page_result.vehicle_links - if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids - ) - page_new = len(page_result.vehicle_links) - page_known - ratio = page_known / len(page_result.vehicle_links) - if ratio >= threshold and page_new == 0: - logger.info( - "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", - page_number, - ratio * 100, - page_known, - len(page_result.vehicle_links), - threshold * 100, - ) - early_stopped = True - break - - fresh_urls = page_urls - page_skipped = 0 - if effective_only_new: - fresh_urls, page_skipped = self._filter_known_urls(page_urls) - skipped_existing += page_skipped - - if limit is not None and limit > 0: - remaining_limit = max(0, limit - total - len(pending_urls)) - if remaining_limit <= 0: - break - fresh_urls = fresh_urls[:remaining_limit] - - pending_urls.extend(fresh_urls) - total += len(fresh_urls) - - logger.info( - "Page %d: %d links, %d new, %d known (total new: %d/%s)", - page_number, - len(page_urls), - len(fresh_urls), - page_skipped, - total, - limit if limit is not None else "∞", - ) - - # Прогресс каждые 10 страниц на уровне WARNING. - if page_number % 10 == 0: - logger.warning( - "sync_listing progress: pages=%d, discovered=%d, upserted=%d, failed=%d, pending=%d", - page_number, - total, - cars_upserted, - cars_failed, - len(pending_urls), - ) - - while len(pending_urls) >= batch_size: - batch_urls = pending_urls[:batch_size] - if not _process_pending_batch(batch_urls, cars_upserted + cars_failed): - pending_urls = [] - break - pending_urls = pending_urls[batch_size:] - - if limit is not None and limit > 0 and total >= limit: - break - if ( - self.settings.listing.collect_current_page_only - or not self.settings.listing.include_pagination - or not page_result.next_page_detected - ): - break - try: - with _PageOpWatchdog(_PAGE_NEXT_TIMEOUT_S, f"next page {page_number + 1}"): - next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1) - except (PageOperationTimeoutError, PlaywrightError) as nav_exc: - logger.warning( - "go_to_next_page stalled/failed at page %d (%s) — will reopen", - page_number + 1, nav_exc, - ) - next_ok = False - if not next_ok: - logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1) - try: - page.close() - except Exception: - pass - page = None - try: - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number + 1, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=max(page_number + 1, 500), - ) - pages_since_rotation = 0 - except ListingResumeError as resume_exc: - logger.warning( - "Cannot resume at page %d (%s) — treating pagination as exhausted", - page_number + 1, resume_exc, - ) - break - - if pending_urls: - _process_pending_batch(pending_urls, cars_upserted + cars_failed) - - # Финальный drain последнего in-flight батча. - _drain_pending_future() - - # Обрабатываем отложенные browser fallback'и (Playwright теперь свободен). - if deferred_fallbacks: - fb_pages = min(len(deferred_fallbacks), self._FALLBACK_PARALLEL_PAGES) - logger.info( - "Processing %d deferred browser fallbacks (%d parallel pages)", - len(deferred_fallbacks), fb_pages, - ) - try: - fb_results = self._browser_fallback_parallel( - deferred_fallbacks, len(deferred_fallbacks), fb_pages, - ) - fb_records = fb_results.get("records", []) - if fb_records: - try: - upsert_result = self.persistence.upsert_cars_batch(fb_records) - cars_upserted += upsert_result["inserted"] + upsert_result["updated"] - images_upserted += upsert_result["images_upserted"] - except Exception as exc: - logger.error("Deferred fallback upsert failed: %s", exc) - cars_failed += len(fb_records) - cars_failed += fb_results.get("cars_failed", 0) - failures.extend(fb_results.get("failures", [])) - except Exception as fb_exc: - logger.error("Deferred fallback batch failed: %s", fb_exc) - cars_failed += len(deferred_fallbacks) - - logger.warning( - "sync_listing final progress: pages=%d, discovered=%d, upserted=%d, failed=%d", - len(pages_info), - total, - cars_upserted, - cars_failed, - ) - finally: - # Ensure no future is left dangling on exception path. - try: - _drain_pending_future() - except Exception: - pass - batch_executor.shutdown(wait=True) - if page is not None: - page.close() - - return { - "listing": { - "status": "ok", - "listing_url": self.settings.listing.cars_url, - "applied_filters": applied_filters, - "pages_collected": len(pages_info), - "vehicles_collected": len(all_raw_urls), - "vehicle_urls": all_raw_urls, - "early_stopped": early_stopped, - "truncated_by_time_budget": truncated_by_time_budget, - "pages": pages_info, - }, - "total": total, - "skipped_existing": skipped_existing, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "failures": failures, - "all_listing_origin_urls": all_listing_origin_urls, - } - - def _get_page(self) -> Page: - if not self.context: - self._new_context() - return self.context.new_page() - - @retryable(max_attempts=3, jitter_seconds=0.25) - def scrape_vehicle_detail(self, vehicle_url: str): - page = self._get_page() - try: - return self._scrape_on_page(page, vehicle_url) - finally: - page.close() - - _JS_EXTRACT = """ - () => { - try { - const scripts = document.querySelectorAll('script:not([src])'); - for (const s of scripts) { - const t = s.textContent || ''; - if (!t.includes('inventoryView') || !t.includes('attributes')) continue; - const start = t.indexOf('{'); - if (start < 0) continue; - let depth = 0, end = -1; - for (let i = start; i < t.length; i++) { - if (t[i] === '{') depth++; - else if (t[i] === '}') { depth--; if (depth === 0) { end = i; break; } } - } - if (end < 0) continue; - try { - const obj = JSON.parse(t.slice(start, end + 1)); - const iv = obj.inventoryView; - if (!iv || !iv.attributes) continue; - const attr = iv.attributes; - const imgs = (iv.imageDimensions && iv.imageDimensions.keys && iv.imageDimensions.keys.$values) - ? iv.imageDimensions.keys.$values : []; - const bid = (obj.auctionInformation && obj.auctionInformation.biddingInformation) - ? obj.auctionInformation.biddingInformation : {}; - const prebid = (obj.auctionInformation && obj.auctionInformation.prebidInformation) - ? obj.auctionInformation.prebidInformation : {}; - return { - ok: true, - SalvageId: attr.SalvageId || '', - StockNumber: attr.StockNumber || '', - Year: attr.Year || '', - Make: attr.Make || '', - Model: attr.Model || '', - Series: attr.Series || '', - BodyStyleName: attr.BodyStyleName || '', - Cylinders: attr.Cylinders || '', - DriveLineTypeDesc: attr.DriveLineTypeDesc || '', - EngineSize: (attr.EngineInformation || attr.EngineSize || '').trim(), - FuelTypeCode: attr.FuelTypeCode || '', - Transmission: attr.Transmission || '', - ExteriorColor: attr.ExteriorColor || '', - PrimaryDamageDesc: attr.PrimaryDamageDesc || '', - SecondaryDamageDesc: attr.SecondaryDamageDesc || '', - ODOValue: attr.ODOValue || '', - ODOBrand: attr.ODOBrand || '', - RunAndDrive: attr.RunAndDrive || '', - Keys: attr.Keys || '', - BranchName: attr.BranchName || '', - AuctionDateTime: attr.AuctionDateTime || '', - Title: attr.Title || '', - TitleBrand: attr.TitleBrand || '', - TitleCode: attr.TitleCode || '', - EstRepairCost: attr.EstRepairCost || '', - VehicleGrade: attr.VehicleGrade || '', - highBidAmount: prebid.highBidAmount || bid.highBidAmount || '', - buyNowPrice: prebid.buyNowPrice || bid.buyNowPrice || '', - acv: attr.ProviderACV || '', - BranchNumber: attr.BranchNumber || '', - imageKeys: imgs.map(i => i.k || '').filter(Boolean), - }; - } catch (_) { continue; } - } - } catch (_) {} - return { ok: false }; - } - """ - - @staticmethod - def _build_car_from_js(js_data: dict, vehicle_url: str) -> dict: - if not js_data or not js_data.get("ok"): - return {} - - brnch = str(js_data.get("BranchNumber", "") or "").strip() - img_keys = js_data.get("imageKeys") or [] - image_urls = [ - f"https://vis.iaai.com/resizer?imageKeys={k}&width=845&height=633" - for k in img_keys - ] - - return { - "source_url": vehicle_url, - "lot_number": js_data.get("StockNumber") or js_data.get("SalvageId"), - "year": js_data.get("Year"), - "make": js_data.get("Make"), - "model": js_data.get("Model"), - "trim": js_data.get("Series"), - "body_type": js_data.get("BodyStyleName"), - "cylinders": js_data.get("Cylinders"), - "drive": js_data.get("DriveLineTypeDesc"), - "engine": js_data.get("EngineSize"), - "fuel_type": js_data.get("FuelTypeCode"), - "gearbox": js_data.get("Transmission"), - "color": js_data.get("ExteriorColor"), - "primary_damage": js_data.get("PrimaryDamageDesc"), - "secondary_damage": js_data.get("SecondaryDamageDesc"), - "odometer": js_data.get("ODOValue"), - "run_and_drive": js_data.get("RunAndDrive"), - "keys": js_data.get("Keys"), - "location": js_data.get("BranchName"), - "auction_date": js_data.get("AuctionDateTime"), - "title": js_data.get("Title"), - "current_bid": js_data.get("highBidAmount"), - "buy_now": js_data.get("buyNowPrice"), - "actual_cash_value": js_data.get("acv"), - "estimated_repair_cost": js_data.get("EstRepairCost"), - "image_urls": image_urls, - } - - @staticmethod - def _build_payload_insights(vehicle_summary: dict) -> dict: - return { - "vehicle_core": vehicle_summary, - "pricing": { - "buy_now": vehicle_summary.get("buy_now"), - "current_bid": vehicle_summary.get("current_bid"), - "actual_cash_value": vehicle_summary.get("actual_cash_value"), - "estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"), - "currency": "USD", - }, - "bids": {"amount": vehicle_summary.get("current_bid"), "currency": "USD"}, - "damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")}, - "auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")}, - "images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []}, - } - - def _scrape_on_page(self, page: Page, vehicle_url: str): - trace_id = self._new_trace_id("scrape") - started_at = time.perf_counter() - - if self.settings.block_resources: - BrowserFactory.enable_resource_blocking(page) - - capture = NetworkCapture(self.settings) - capture.attach(page, origin_url=vehicle_url) - - page.goto(vehicle_url, wait_until="commit", timeout=60_000) - - try: - page.wait_for_load_state("domcontentloaded", timeout=5_000) - except PlaywrightTimeoutError: - pass - - js_data: dict = {} - try: - js_data = page.evaluate(self._JS_EXTRACT) or {} - except Exception: - pass - - if js_data.get("ok"): - vehicle_summary = self._build_car_from_js(js_data, vehicle_url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise SiteStructureChangedError(f"JS extraction returned no vehicle identity for {vehicle_url}") - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - network_dump = capture.export() - result = { - "trace_id": trace_id, - "source_url": vehicle_url, - "fetched_at_epoch": int(time.time()), - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "network": network_dump, - "vehicle_summary": vehicle_summary, - "payload_insights": {}, - "embedded_json": [], - "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, - "access_notes": {}, - "db_record": db_record.model_dump(mode="json"), - } - if self.settings.raw_output_json: - save_to_json(network_dump, self.settings.raw_output_json) - return result - - # Резервный путь: полный HTML-парсинг. - try: - page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=400) - except PlaywrightTimeoutError: - pass - - html = page.content() - try: - dom_text = page.evaluate("() => document.body?.textContent || ''") - except Exception: - dom_text = "" - network_dump = capture.export() - parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) - self._raise_if_blocked_or_incomplete(parsed, vehicle_url) - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=parsed.get("vehicle_summary", {}), - payload_insights=parsed.get("payload_insights", {}), - ) - - result = { - "trace_id": trace_id, - "source_url": vehicle_url, - "fetched_at_epoch": int(time.time()), - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "network": network_dump, - **parsed, - "db_record": db_record.model_dump(mode="json"), - } - - if self.settings.raw_output_json: - save_to_json(network_dump, self.settings.raw_output_json) - return result - - @staticmethod - def _extract_iaai_json_from_html(html: str, vehicle_url: str) -> dict | None: - """Извлекает IAAI inventoryView.attributes из HTML без браузера. - - Использует json.JSONDecoder.raw_decode для быстрого поиска JSON - вместо посимвольного сканирования скобок. Если получившийся срез - целиком отдаётся декодеру — пробуем orjson (3-5× быстрее stdlib). - """ - search = "inventoryView" - pos = html.find(search) - if pos < 0: - return None - - script_start = html.rfind("", script_start) - if tag_end < 0: - return None - content_start = html.find("{", tag_end) - if content_start < 0: - return None - - # Быстрый путь: вырезаем всё до , пытаемся orjson; иначе stdlib. - obj: dict | None = None - if _HAS_ORJSON: - script_end = html.find("", content_start) - if script_end > content_start: - # Подрезаем хвост до последней закрывающей скобки JSON. - snippet = html[content_start:script_end].rstrip().rstrip(";") - try: - obj = _orjson.loads(snippet) - except Exception: - obj = None - - if obj is None: - decoder = json.JSONDecoder() - try: - obj, _ = decoder.raw_decode(html, content_start) - except (json.JSONDecodeError, ValueError): - return None - - iv = obj.get("inventoryView") - if not iv or not iv.get("attributes"): - return None - - attr = iv["attributes"] - imgs = [] - try: - imgs = iv.get("imageDimensions", {}).get("keys", {}).get("$values", []) or [] - except Exception: - pass - - bid = {} - prebid = {} - try: - ai = obj.get("auctionInformation", {}) - bid = ai.get("biddingInformation", {}) or {} - prebid = ai.get("prebidInformation", {}) or {} - except Exception: - pass - - img_keys = [i.get("k", "") for i in imgs if i.get("k")] - return { - "ok": True, - "SalvageId": attr.get("SalvageId", ""), - "StockNumber": attr.get("StockNumber", ""), - "Year": attr.get("Year", ""), - "Make": attr.get("Make", ""), - "Model": attr.get("Model", ""), - "Series": attr.get("Series", ""), - "BodyStyleName": attr.get("BodyStyleName", ""), - "Cylinders": attr.get("Cylinders", ""), - "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), - "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), - "FuelTypeCode": attr.get("FuelTypeCode", ""), - "Transmission": attr.get("Transmission", ""), - "ExteriorColor": attr.get("ExteriorColor", ""), - "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), - "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), - "ODOValue": attr.get("ODOValue", ""), - "ODOBrand": attr.get("ODOBrand", ""), - "RunAndDrive": attr.get("RunAndDrive", ""), - "Keys": attr.get("Keys", ""), - "BranchName": attr.get("BranchName", ""), - "AuctionDateTime": attr.get("AuctionDateTime", ""), - "Title": attr.get("Title", ""), - "TitleBrand": attr.get("TitleBrand", ""), - "TitleCode": attr.get("TitleCode", ""), - "EstRepairCost": attr.get("EstRepairCost", ""), - "VehicleGrade": attr.get("VehicleGrade", ""), - "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), - "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), - "acv": attr.get("ProviderACV", ""), - "BranchNumber": attr.get("BranchNumber", ""), - "imageKeys": img_keys, - } - - def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord: - if not self.context: - self._new_context() - assert self.context is not None - last_error: Exception | None = None - max_attempts = max(1, self.settings.fast_path_max_attempts) - timeout_ms = max(1000, self.settings.fast_path_timeout_ms) - for attempt in range(1, max_attempts + 1): - try: - response = self.context.request.get(vehicle_url, timeout=timeout_ms) - if not response.ok: - raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") - - html = response.text() - - js_data = self._extract_iaai_json_from_html(html, vehicle_url) - if not js_data: - raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") - - vehicle_summary = self._build_car_from_js(js_data, vehicle_url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - return CarRecord.model_validate(db_record.model_dump(mode="json")) - except Exception as exc: - last_error = exc - if attempt < max_attempts: - time.sleep(0.2) - - if last_error is not None: - raise last_error - raise RuntimeError(f"HTTP fast-path failed for {vehicle_url}") - - def _cookie_header_for_context(self) -> str: - if not self.context: - return "" - try: - cookies = self.context.cookies() - except Exception: - return "" - pairs = [ - f"{item.get('name')}={item.get('value')}" - for item in cookies - if item.get("name") and item.get("value") is not None - ] - return "; ".join(pairs) - - def _scrape_via_raw_http( - self, - vehicle_url: str, - *, - cookie_header: str, - user_agent: str, - ) -> CarRecord: - """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive). - - Заголовок Accept-Encoding включает gzip/deflate/br — IAAI отдаёт HTML - со сжатием 5-10×, что снижает байты по сети и время декода. - urllib3 сам прозрачно распаковывает gzip и deflate; для br нужен - пакет `brotli` (он в зависимостях). Если brotli не установлен — он - просто не появится в Accept-Encoding и сервер ответит gzip. - """ - headers = { - "User-Agent": user_agent, - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.9", - "Accept-Encoding": self._ACCEPT_ENCODING, - "Cache-Control": "no-cache", - "Pragma": "no-cache", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1", - } - if cookie_header: - headers["Cookie"] = cookie_header - - # decode_content=True — urllib3 сам распакует gzip/deflate/br. - response = self._http_pool.request( - "GET", vehicle_url, headers=headers, decode_content=True, - ) - if response.status >= 400: - raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") - html = response.data.decode("utf-8", errors="ignore") - - js_data = self._extract_iaai_json_from_html(html, vehicle_url) - if not js_data: - raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") - - vehicle_summary = self._build_car_from_js(js_data, vehicle_url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - return CarRecord.model_validate(db_record.model_dump(mode="json")) - - def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"): - trace_id = self._new_trace_id("sync-vehicle") - started_at = time.perf_counter() - self.persistence.create_tables() - run_id = self.persistence.start_sync_run(lane=lane) - ids_fetched = 1 - cars_upserted = 0 - cars_failed = 0 - images_upserted = 0 - status = "failed" - error_summary = None - try: - scrape_result = self.scrape_vehicle_detail(vehicle_url) - db_record = scrape_result.get("db_record") - if not db_record: - raise RuntimeError("Scrape result does not contain db_record") - record = CarRecord.model_validate(db_record) - upsert = self.persistence.upsert_car(record) - cars_upserted = 1 - images_upserted = int(upsert.get("images_upserted", 0)) - status = "success" - return { - "trace_id": trace_id, - "status": status, - "run_id": run_id, - "vehicle_url": vehicle_url, - "db_action": upsert.get("action"), - "images_upserted": images_upserted, - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "db_record": db_record, - } - except Exception as exc: - cars_failed = 1 - status = "failed" - error_summary = str(exc) - raise - finally: - self.persistence.finish_sync_run( - run_id, - status=status, - ids_fetched=ids_fetched, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - images_upserted=images_upserted, - error_summary=error_summary, - ) - - # ── Настройки sync_batch ── - _HTTP_MICRO_BATCH = 25 # URL за микро-батч - _HTTP_MAX_RETRIES = 2 # Повторов на URL до перехода в браузер - _HTTP_RETRY_DELAYS = (0.4, 1.0) # Задержки между повторами - _FALLBACK_PARALLEL_PAGES = 4 # Параллельных вкладок для fallback - _FALLBACK_NAV_TIMEOUT_MS = 8000 # Таймаут навигации fallback - # gzip+deflate: нативно в urllib3 (zlib, ~0 CPU). Brotli отключён намеренно — - # python-brotli декодирует на CPU и под 32 параллельными потоками становится - # узким местом (хуже, чем гонять чуть больше байт по сети). - _ACCEPT_ENCODING = "gzip, deflate" - - def _browser_fallback_parallel( - self, - fallback_urls: list[tuple[str, int]], - total: int, - n_pages: int, - ) -> dict: - records: list[CarRecord] = [] - failures: list[dict[str, str]] = [] - cars_failed = 0 - protection_events = 0 - - slices: list[list[tuple[str, int]]] = [[] for _ in range(n_pages)] - for i, item in enumerate(fallback_urls): - slices[i % n_pages].append(item) - - def _process_slice(url_slice: list[tuple[str, int]]) -> dict: - local_records: list[CarRecord] = [] - local_failures: list[dict[str, str]] = [] - local_failed = 0 - local_protection = 0 - page: Page | None = None - try: - for url, global_idx in url_slice: - if page is None: - page = self._get_page() - if self.settings.block_resources: - BrowserFactory.enable_resource_blocking(page) - - try: - page.goto( - url, - wait_until="commit", - timeout=self._FALLBACK_NAV_TIMEOUT_MS, - ) - except Exception as exc: - if self._is_protection_or_network_error(exc): - local_protection += 1 - local_failed += 1 - local_failures.append({"vehicle_url": url, "error": str(exc)}) - logger.error("[%d/%d] Failed to open %s: %s", global_idx, total, url, exc) - try: - page.close() - except Exception: - pass - page = None - continue - - try: - js_data: dict = {} - try: - js_data = page.evaluate(self._JS_EXTRACT) or {} - except Exception: - pass - - if not js_data.get("ok"): - try: - page.wait_for_load_state("domcontentloaded", timeout=3_000) - except PlaywrightTimeoutError: - pass - try: - js_data = page.evaluate(self._JS_EXTRACT) or {} - except Exception: - pass - - if js_data.get("ok"): - vehicle_summary = self._build_car_from_js(js_data, url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise SiteStructureChangedError( - f"JS extraction returned no vehicle identity for {url}" - ) - db_record = self.car_mapper.map_to_car_record( - vehicle_url=url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - else: - try: - page.wait_for_selector( - "#VehicleDetailViewModel, .veh-details, .vehicle-details, " - "[data-uname='vehicleDetailPage']", - timeout=400, - ) - except PlaywrightTimeoutError: - pass - page_html = page.content() - try: - dom_text = page.evaluate("() => document.body?.textContent || ''") - except Exception: - dom_text = "" - network_dump = { - "requests": [], - "json_responses": [], - "capture_limits": {}, - } - parsed = self.vehicle_parser.normalize(url, page_html, dom_text, network_dump) - self._raise_if_blocked_or_incomplete(parsed, url) - db_record = self.car_mapper.map_to_car_record( - vehicle_url=url, - vehicle_summary=parsed.get("vehicle_summary", {}), - payload_insights=parsed.get("payload_insights", {}), - ) - - record = CarRecord.model_validate(db_record.model_dump(mode="json")) - local_records.append(record) - logger.debug( - "[%d/%d] Parsed %s %s %s (fallback)", - global_idx, total, record.brand, record.model, record.year or "?", - ) - except Exception as exc: - if self._is_protection_or_network_error(exc): - local_protection += 1 - local_failed += 1 - local_failures.append({"vehicle_url": url, "error": str(exc)}) - logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) - finally: - if page is not None: - try: - page.close() - except Exception: - pass - return { - "records": local_records, - "failures": local_failures, - "cars_failed": local_failed, - "protection_events": local_protection, - } - - # Одна страница — в главном потоке. - if n_pages == 1: - res = _process_slice(slices[0] if slices else []) - records.extend(res["records"]) - failures.extend(res["failures"]) - cars_failed += res["cars_failed"] - protection_events += res["protection_events"] - else: - # Несколько страниц — параллельно, каждый поток со своей Page. - with ThreadPoolExecutor(max_workers=n_pages) as executor: - futures = [executor.submit(_process_slice, s) for s in slices if s] - for fut in as_completed(futures): - res = fut.result() - records.extend(res["records"]) - failures.extend(res["failures"]) - cars_failed += res["cars_failed"] - protection_events += res["protection_events"] - - return { - "records": records, - "failures": failures, - "cars_failed": cars_failed, - "protection_events": protection_events, - } - - cookie_header_override: str | None = None, - def sync_batch( - self, - vehicle_urls: list[str], - lane: str = "iaai_cars", - parallel_tabs: int | None = None, - *, - skip_browser_fallback: bool = False, - cookie_header_override: str | None = None, - ) -> dict: - trace_id = self._new_trace_id("sync-batch") - started_at = time.perf_counter() - num_workers = parallel_tabs or self.settings.parallel_tabs - num_workers = min(num_workers, len(vehicle_urls), 48) - - cars_upserted = 0 - cars_failed = 0 - images_upserted = 0 - records: list[CarRecord] = [] - # cookie_header_override передаётся из главного потока при pipeline-вызове — - # вычисление cookie_header из background thread'а ломает Playwright sync greenlet. - cookie_header = ( - cookie_header_override - if cookie_header_override is not None - else self._cookie_header_for_context() - - http_successes = 0 - http_fallbacks = 0 - protection_events = 0 - - total = len(vehicle_urls) - logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) - - # cookie_header_override передаётся из главного потока при pipeline-вызове — - # вычисление cookie_header из background thread'а ломает Playwright sync greenlet. - cookie_header = ( - cookie_header_override - if cookie_header_override is not None - else self._cookie_header_for_context() - ) - # UA должен совпадать с UA Playwright-контекста, иначе Imperva видит - # «cookies от Chrome, ходим под Firefox» — типичный сигнал бота. - user_agent = self.settings.fingerprint.user_agent - - # ── Фаза 1: HTTP fast-path ── - fallback_urls: list[tuple[str, int]] = [] - - def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: - idx, url = idx_url - # Небольшой анти-бёрст джиттер: разносим старт запросов в пуле, - # чтобы N воркеров не били в одну TLS-/PX-волну. - time.sleep(random.uniform(0.0, 0.15)) - last_exc: Exception | None = None - for attempt in range(1 + self._HTTP_MAX_RETRIES): - try: - return idx, self._scrape_via_raw_http( - url, - cookie_header=cookie_header, - user_agent=user_agent, - ) - except Exception as exc: - last_exc = exc - if attempt < self._HTTP_MAX_RETRIES: - time.sleep(self._HTTP_RETRY_DELAYS[min(attempt, len(self._HTTP_RETRY_DELAYS) - 1)]) - return idx, last_exc # type: ignore[return-value] - - indexed_urls = list(enumerate(vehicle_urls)) - with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor: - for idx, result in executor.map(_fetch_one_with_retry, indexed_urls): - if isinstance(result, Exception): - http_fallbacks += 1 - logger.debug( - "[%d/%d] HTTP fast-path failed for %s: %s", - idx + 1, total, vehicle_urls[idx], result, - ) - fallback_urls.append((vehicle_urls[idx], idx + 1)) - else: - records.append(result) - http_successes += 1 - logger.debug( - "[%d/%d] Parsed %s %s %s (raw HTTP)", - idx + 1, total, result.brand, result.model, result.year or "?", - ) - - logger.info( - "HTTP phase done: %d OK, %d fallback (%.1fs)", - http_successes, http_fallbacks, time.perf_counter() - started_at, - ) - - # ── Фаза 2: браузерный fallback ── - # При skip_browser_fallback=True (pipeline-режим) возвращаем fallback_urls - # вызывающему — Playwright занят сбором листинга в главном потоке, - # его нельзя трогать из фонового executor'а. - deferred_fallback_urls: list[tuple[str, int]] = [] - if fallback_urls: - if skip_browser_fallback: - deferred_fallback_urls = fallback_urls - logger.info( - "Deferring browser fallback: %d/%d URLs (pipeline mode)", - len(fallback_urls), total, - ) - else: - fb_pages = min(len(fallback_urls), self._FALLBACK_PARALLEL_PAGES) - logger.info( - "Fallback browser mode: %d/%d URLs, %d parallel pages", - len(fallback_urls), total, fb_pages, - ) - fb_results = self._browser_fallback_parallel(fallback_urls, total, fb_pages) - records.extend(fb_results["records"]) - cars_failed += fb_results["cars_failed"] - protection_events += fb_results["protection_events"] - failures.extend(fb_results["failures"]) - - # ── Фаза 3: запись в БД ── - if records: - seen_origins: set[str] = set() - unique_records: list[CarRecord] = [] - for rec in records: - key = rec.origin_id or rec.origin_url - if key not in seen_origins: - seen_origins.add(key) - unique_records.append(rec) - if len(unique_records) < len(records): - logger.info("Dedup before DB: %d → %d", len(records), len(unique_records)) - records = unique_records - - try: - batch_result = self.persistence.upsert_cars_batch(records) - cars_upserted = batch_result["inserted"] + batch_result["updated"] - images_upserted = batch_result["images_upserted"] - except Exception as exc: - logger.error("Batch upsert failed: %s", exc) - cars_failed += len(records) - - status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - return { - "trace_id": trace_id, - "status": status, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "http_successes": http_successes, - "http_fallbacks": http_fallbacks, - "protection_events": protection_events, - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "failures": failures, - "deferred_fallback_urls": deferred_fallback_urls, - } - - def sync_listing( - self, - make: str | None = None, - model: str | None = None, - lane: str = "iaai_cars", - limit: int | None = None, - only_new: bool | None = None, - listing_url: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - ): - # runtime_config — дефолты; CLI/API аргументы приоритетнее. - rc = self.runtime_config.sync - if limit is None and rc.limit is not None: - limit = rc.limit - if only_new is None and rc.only_new is not None: - only_new = rc.only_new - if lane == "iaai_cars" and rc.lane is not None: - lane = rc.lane - - trace_id = self._new_trace_id("sync-listing") - started_at = time.perf_counter() - self.persistence.create_tables() - run_id = self.persistence.start_sync_run(lane=lane) - cars_upserted = 0 - cars_failed = 0 - cars_filtered = 0 - images_upserted = 0 - total = 0 - skipped_existing = 0 - is_partial_scan = True - failures: list[dict[str, str]] = [] - listing: dict = {} - - try: - effective_only_new = self.settings.sync_only_new if only_new is None else only_new - stream_result = self._sync_listing_streaming( - make=make, - model=model, - lane=lane, - limit=limit, - effective_only_new=effective_only_new, - started_at=started_at, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) - listing = stream_result["listing"] - total = stream_result["total"] - skipped_existing = stream_result["skipped_existing"] - cars_upserted = stream_result["cars_upserted"] - cars_failed = stream_result["cars_failed"] - images_upserted = stream_result["images_upserted"] - failures.extend(stream_result["failures"]) - all_listing_origin_urls = stream_result["all_listing_origin_urls"] - - logger.info("Streaming sync processed %d vehicles", total) - - # Помечаем проданные авто, исчезнувшие из листинга (только при полном скане). - is_partial_scan = ( - effective_only_new - or (limit is not None and limit > 0) - or listing.get("early_stopped", False) - or listing.get("truncated_by_time_budget", False) - ) - if all_listing_origin_urls and not is_partial_scan: - try: - sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) - if sold_count: - logger.info("Marked %d cars as sold", sold_count) - except Exception as exc: - logger.warning("Failed to mark sold cars: %s", exc) - elif is_partial_scan: - logger.debug("Skipping mark_sold: partial/incremental scan (only_new=%s, limit=%s, early_stopped=%s)", - effective_only_new, limit, listing.get("early_stopped", False)) - except Exception as exc: - if not failures: - failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) - logger.error("sync_listing failed: %s (partial progress: %d upserted)", exc, cars_upserted) - finally: - status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None - self.persistence.finish_sync_run( - run_id, - status=status, - ids_fetched=total, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - images_upserted=images_upserted, - error_summary=error_summary, - ) - - logger.info( - "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images", - run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted, - ) - return { - "trace_id": trace_id, - "status": status, - "run_id": run_id, - # partial_success допустим — отдельные машины могли не спарситься, - # это не повод повторять весь bootstrap. - "full_scan_completed": (not is_partial_scan) and status in ("success", "partial_success"), - "only_new_effective": effective_only_new, - "listing": listing, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "cars_filtered": cars_filtered, - "images_upserted": images_upserted, - "skipped_existing": skipped_existing, - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "failures": failures, - } - - def sync_listing_segmented( - self, - segments: list[dict[str, Any]], - lane: str = "iaai_cars", - only_new: bool | None = None, - start_segment: int = 0, - start_page: int = 1, - progress_callback: Callable[[int], None] | None = None, - ) -> dict[str, Any]: - """Итеративный sync_listing по списку сегментов (бренд / бренд+годы). - - Args: - segments: список dict с ключами make, year_min, year_max. - start_segment: индекс сегмента для resume (0-based). - start_page: не используется (остаётся для обратной совместимости API). - progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента. - """ - trace_id = self._new_trace_id("sync-segmented") - started_at = time.perf_counter() - base_url = self.settings.listing.cars_url - _ = start_page # обратная совместимость — page-level resume удалён - - total_cars_upserted = 0 - total_cars_failed = 0 - total_images_upserted = 0 - total_skipped = 0 - total_discovered = 0 - all_failures: list[dict[str, str]] = [] - segment_results: list[dict[str, Any]] = [] - completed_all = True - - logger.warning( - "Starting segmented sync: %d segments, resume from segment=%d", - len(segments), start_segment, - ) - - for seg_idx in range(start_segment, len(segments)): - seg = segments[seg_idx] - seg_make = seg.get("make") - seg_year_min = seg.get("year_min") - seg_year_max = seg.get("year_max") - seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None - - seg_label = f"{seg_make or 'ALL'}" - if seg_year_min is not None or seg_year_max is not None: - seg_label += f" ({seg_year_min}-{seg_year_max})" - - logger.warning( - "Segment %d/%d: %s", - seg_idx + 1, len(segments), seg_label, - ) - - try: - result = self.sync_listing( - make=None if seg_url else seg_make, - model=None, - lane=lane, - only_new=only_new, - listing_url=seg_url, - year_min=seg_year_min, - year_max=seg_year_max, - ) - total_cars_upserted += result.get("cars_upserted", 0) - total_cars_failed += result.get("cars_failed", 0) - total_images_upserted += result.get("images_upserted", 0) - total_skipped += result.get("skipped_existing", 0) - total_discovered += result.get("listing", {}).get("vehicles_collected", 0) - all_failures.extend(result.get("failures", [])) - segment_results.append({ - "segment": seg, - "segment_index": seg_idx, - "status": result.get("status"), - "full_scan_completed": bool(result.get("full_scan_completed", False)), - "cars_upserted": result.get("cars_upserted", 0), - "cars_failed": result.get("cars_failed", 0), - "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), - }) - - segment_done = bool(result.get("full_scan_completed", False)) - if not segment_done: - completed_all = False - - # Сегмент полностью пройден — фиксируем чекпоинт, даже если часть машин failed. - if segment_done and progress_callback is not None: - try: - progress_callback(seg_idx) - except Exception: - logger.warning( - "Failed to persist segment checkpoint for segment=%d", - seg_idx, exc_info=True, - ) - - logger.warning( - "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d", - seg_idx + 1, len(segments), seg_label, - result.get("cars_upserted", 0), - result.get("cars_failed", 0), - result.get("listing", {}).get("vehicles_collected", 0), - ) - except Exception as exc: - logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc) - all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)}) - completed_all = False - # Продолжаем оставшиеся сегменты — одна ошибка не должна убивать весь прогон - continue - - elapsed = round(time.perf_counter() - started_at, 3) - status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed" - - logger.warning( - "Segmented sync done: %d/%d segments, upserted=%d, failed=%d, discovered=%d, elapsed=%.1fs", - len(segment_results), len(segments), - total_cars_upserted, total_cars_failed, total_discovered, elapsed, - ) - - return { - "trace_id": trace_id, - "status": status, - # Bootstrap считается завершённым если все сегменты пройдены, - # даже если часть машин failed (они будут обновлены в следующих циклах). - "full_scan_completed": completed_all, - "segments_total": len(segments), - "segments_completed": len(segment_results), - "cars_upserted": total_cars_upserted, - "cars_failed": total_cars_failed, - "images_upserted": total_images_upserted, - "skipped_existing": total_skipped, - "total_discovered": total_discovered, - "elapsed_seconds": elapsed, - "failures": all_failures, - "segment_results": segment_results, - } - - def run_scheduled(self) -> None: - """Standalone-планировщик (в продакшене используется Celery beat).""" - def _handle_shutdown(signum, frame): - logger.info("Received signal %s, shutting down gracefully...", signum) - self._shutdown_requested = True - - signal.signal(signal.SIGINT, _handle_shutdown) - signal.signal(signal.SIGTERM, _handle_shutdown) - - interval = self.settings.scheduler_interval_minutes * 60 - logger.info( - "Scheduler started: syncing every %d minutes", - self.settings.scheduler_interval_minutes, - ) - cycle = 0 - while not self._shutdown_requested: - cycle += 1 - logger.info("Scheduler cycle #%d starting", cycle) - start = time.time() - try: - if self.context: - try: - self.context.close() - except PlaywrightError: - pass - self.context = None - - if self.browser is None or self.playwright is None: - logger.info("Browser/Playwright not available, re-initializing...") - self.close() - self.__enter__() - - result = self.sync_listing() - elapsed = time.time() - start - logger.info( - "Cycle #%d done in %.1fs: %d upserted, %d failed", - cycle, elapsed, - result.get("cars_upserted", 0), - result.get("cars_failed", 0), - ) - except Exception as exc: - elapsed = time.time() - start - logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) - try: - self.close() - except Exception: - pass - try: - self.__enter__() - except Exception as reinit_exc: - logger.error("Failed to re-initialize browser: %s", reinit_exc) - - if self._shutdown_requested: - break - - sleep_time = max(0, interval - (time.time() - start)) - if sleep_time > 0: - logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) - slept = 0.0 - while slept < sleep_time and not self._shutdown_requested: - chunk = min(5.0, sleep_time - slept) - time.sleep(chunk) - slept += chunk - - logger.info("Scheduler stopped gracefully after %d cycles.", cycle) diff --git a/iaai_scraper/storage/__init__.py b/iaai_scraper/storage/__init__.py deleted file mode 100644 index c9c2ef6..0000000 --- a/iaai_scraper/storage/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__: list[str] = [] diff --git a/iaai_scraper/worker/tasks.py b/iaai_scraper/worker/tasks.py deleted file mode 100644 index bde465a..0000000 --- a/iaai_scraper/worker/tasks.py +++ /dev/null @@ -1,648 +0,0 @@ -# Задачи Celery для синхронизации автомобилей и листинга IAAI. - -from concurrent.futures import ThreadPoolExecutor -import logging -from threading import Event, Thread -import time -import uuid - -from billiard.exceptions import SoftTimeLimitExceeded -from celery import shared_task -from redis import Redis - -from ..core.config import Settings, parse_listing_segments -from ..scraper import IAAIScraper -from ..storage.db import PersistenceService - -logger = logging.getLogger("iaai_scraper.worker.tasks") - -SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing" -SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done" -SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint" -SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60 -SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak" -SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3 -SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60 -SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending" -SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task" - - -def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): - last_exc: Exception | None = None - for attempt in range(1, attempts + 1): - try: - return func() - except Exception as exc: - last_exc = exc - if attempt >= attempts: - break - delay = base_delay_s * (2 ** (attempt - 1)) - logger.warning( - "Operation failed (attempt %d/%d): %s. Retrying in %.1fs", - attempt, - attempts, - exc, - delay, - ) - time.sleep(delay) - if last_exc is not None: - raise last_exc - - -def _run_browser_job(func, *args, **kwargs): - # Браузерный код запускаем в отдельном потоке без активного loop. - # НЕ используем `with` — иначе shutdown(wait=True) заблокирует main thread - # если SoftTimeLimitExceeded прервёт future.result(), а browser thread ещё работает. - settings = Settings() - soft = settings.celery.task_soft_time_limit - hard = settings.celery.task_time_limit - # Таймаут для future.result(): берём hard limit (или soft + 120), чтобы не висеть вечно. - wait_timeout = min(hard, soft + 120) if soft and hard else None - - executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser") - future = executor.submit(func, *args, **kwargs) - try: - result = future.result(timeout=wait_timeout) - except SoftTimeLimitExceeded: - # Отпускаем executor без ожидания — thread умрёт когда Celery убьёт процесс (hard limit). - future.cancel() - executor.shutdown(wait=False, cancel_futures=True) - raise - except TimeoutError: - # future.result(timeout=...) вышел по таймауту — browser thread завис. - future.cancel() - executor.shutdown(wait=False, cancel_futures=True) - raise SoftTimeLimitExceeded("Browser thread did not finish within time limit") - except Exception: - executor.shutdown(wait=False, cancel_futures=True) - raise - else: - executor.shutdown(wait=True) - return result - - -def _sync_listing_lock_ttl_seconds() -> int: - settings = Settings() - soft = settings.celery.task_soft_time_limit - hard = settings.celery.task_time_limit - # Используем clamped hard limit (soft + 120), а не сырой task_time_limit, - # чтобы lock не висел 11 дней при CELERY_TASK_TIME_LIMIT=999999. - effective_hard = min(hard, soft + 120) if soft else hard - return max(effective_hard + 120, 300) - - -def _get_persistence() -> PersistenceService: - settings = Settings() - persistence = PersistenceService(settings) - - def _ping_db() -> None: - with persistence.engine.connect() as conn: - conn.exec_driver_sql("SELECT 1") - - _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0) - return persistence - - -def _get_redis() -> Redis: - settings = Settings() - redis_client = Redis.from_url( - settings.redis.url, - decode_responses=True, - socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, - socket_timeout=settings.redis.socket_timeout_seconds, - health_check_interval=settings.redis.health_check_interval_seconds, - retry_on_timeout=True, - ) - - def _ping_redis() -> None: - redis_client.ping() - - _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0) - return redis_client - - -def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: - try: - acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) - if acquired: - return True - - # Автовосстановление: если lock завис без TTL, считаем stale и пересоздаём. - ttl = redis_client.ttl(key) - if ttl is not None and ttl < 0: - logger.warning("Detected stale lock without TTL, removing: %s", key) - redis_client.delete(key) - return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) - - return False - except Exception as exc: - logger.warning("Failed to acquire lock %s", key, exc_info=True) - return False - - -def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: - try: - refreshed = redis_client.eval( - """ - if redis.call('GET', KEYS[1]) == ARGV[1] then - return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) - end - return 0 - """, - 1, - key, - owner_token, - int(ttl_seconds), - ) - return bool(refreshed) - except Exception as exc: - logger.warning("Failed to refresh lock %s", key, exc_info=True) - return None - - -def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: - try: - redis_client.eval( - """ - if redis.call('GET', KEYS[1]) == ARGV[1] then - return redis.call('DEL', KEYS[1]) - end - return 0 - """, - 1, - key, - owner_token, - ) - except Exception as exc: - logger.warning("Failed to release lock %s", key, exc_info=True) - - -def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool: - try: - inspector = celery_app.control.inspect(timeout=1.0) - snapshots = [ - inspector.active() or {}, - inspector.reserved() or {}, - inspector.scheduled() or {}, - ] - except Exception: - logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True) - return True - - for snapshot in snapshots: - for entries in snapshot.values(): - for entry in entries or []: - task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") - if task_name != SYNC_LISTING_TASK_NAME: - continue - # Исключаем текущую задачу — она не считается "другой запущенной" - entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "") - if exclude_task_id and entry_id == exclude_task_id: - continue - return True - return False - - -def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool: - try: - owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY) - if not owner_token: - return False - except Exception: - logger.warning("Failed to read sync listing lock before cleanup", exc_info=True) - return False - - if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id): - logger.info("sync_listing lock preserved: active task still detected") - return False - - try: - ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY) - redis_client.delete(SYNC_LISTING_LOCK_KEY) - logger.warning( - "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart", - owner_token, - ttl, - ) - return True - except Exception: - logger.warning("Failed to clear orphan sync listing lock", exc_info=True) - return False - - -def _is_full_scan_done(redis_client: Redis) -> bool: - try: - value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY) - except Exception: - logger.warning("Failed to read full scan state", exc_info=True) - return False - return str(value or "").strip() == "1" - - -def _set_full_scan_done(redis_client: Redis, done: bool) -> None: - try: - redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0") - except Exception: - logger.warning("Failed to persist full scan state", exc_info=True) - - -def _load_last_completed_segment(redis_client: Redis) -> int | None: - # Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. - try: - raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY) - except Exception: - logger.warning("Failed to read sync listing checkpoint", exc_info=True) - return None - if raw is None: - return None - try: - value = int(str(raw).strip()) - except (TypeError, ValueError): - logger.warning("Invalid sync listing checkpoint value %r; clearing", raw) - _clear_sync_checkpoint(redis_client) - return None - if value < 0: - _clear_sync_checkpoint(redis_client) - return None - return value - - -def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None: - try: - redis_client.set( - SYNC_LISTING_CHECKPOINT_KEY, - str(int(segment_index)), - ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS, - ) - except Exception: - logger.warning("Failed to save sync listing checkpoint", exc_info=True) - - -def _clear_sync_checkpoint(redis_client: Redis) -> None: - try: - redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY) - except Exception: - logger.warning("Failed to clear sync listing checkpoint", exc_info=True) - - -def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool: - try: - return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds)))) - except Exception: - logger.warning("Failed to set follow-up pending flag", exc_info=True) - return True - - -def _clear_followup_pending(redis_client: Redis) -> None: - try: - redis_client.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY) - except Exception: - logger.warning("Failed to clear follow-up pending flag", exc_info=True) - - -def _bump_bootstrap_failure_streak( - redis_client: Redis, - *, - reason: str, -) -> tuple[int, bool]: - try: - streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)) - redis_client.expire( - SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, - SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, - ) - except Exception: - logger.warning("Failed to update bootstrap failure streak", exc_info=True) - return 0, True - - should_enqueue = streak < SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT - if should_enqueue: - logger.warning( - "Bootstrap failure streak %d/%d recorded (reason=%s)", - streak, - SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, - reason, - ) - else: - logger.error( - "Bootstrap follow-up circuit breaker opened after %d consecutive failures (reason=%s)", - streak, - reason, - ) - return streak, should_enqueue - - -def _clear_bootstrap_failure_streak(redis_client: Redis) -> None: - try: - redis_client.delete(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY) - except Exception: - logger.warning("Failed to clear bootstrap failure streak", exc_info=True) - - -def _start_lock_heartbeat( - redis_client: Redis, - key: str, - owner_token: str, - ttl_seconds: int, -) -> tuple[Event, Thread]: - stop_event = Event() - interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) - - def _heartbeat() -> None: - while not stop_event.wait(interval_seconds): - refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) - if refreshed is False: - logger.warning("Lost sync_listing lock ownership for %s", owner_token) - return - - thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) - thread.start() - return stop_event, thread - - -@shared_task( - name="iaai_scraper.worker.tasks.sync_vehicle_task", - bind=True, - max_retries=2, - default_retry_delay=30, - acks_late=True, -) -def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"): - # Скрапинг и upsert одного автомобиля. - persistence = _get_persistence() - persistence.create_tables() - - try: - def _job(): - with IAAIScraper() as scraper: - return scraper.sync_vehicle(vehicle_url, lane=lane) - - result = _run_browser_job(_job) - logger.info("sync_vehicle_task completed: %s", vehicle_url) - return { - "status": "success", - "vehicle_url": vehicle_url, - "db_action": result.get("db_action"), - "images_upserted": result.get("images_upserted", 0), - } - - except Exception as exc: - logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True) - raise self.retry(exc=exc) - - -@shared_task( - name="iaai_scraper.worker.tasks.sync_listing_task", - bind=True, - max_retries=3, - default_retry_delay=120, - acks_late=True, -) -def sync_listing_task( - self, - make: str | None = None, - model: str | None = None, - lane: str = "iaai_cars", - limit: int | None = None, - only_new: bool | None = None, -): - # Полный цикл: листинг + sync всех найденных машин. - persistence = _get_persistence() - persistence.create_tables() - task_id = self.request.id or "unknown" - owner_token = f"{task_id}:{uuid.uuid4().hex}" - redis_client = _get_redis() - - lock_acquired = False - lock_ttl = _sync_listing_lock_ttl_seconds() - heartbeat_stop: Event | None = None - heartbeat_thread: Thread | None = None - force_bootstrap_full_scan = False - - def _enqueue_bootstrap_followup( - reason: str, - delay_seconds: int = 5, - *, - count_as_failure: bool = False, - ) -> None: - flag_ttl = max(lock_ttl, delay_seconds + 300) - if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl): - logger.info( - "Bootstrap follow-up already pending; skip enqueue (reason=%s)", - reason, - ) - return - if count_as_failure: - _, should_enqueue = _bump_bootstrap_failure_streak(redis_client, reason=reason) - if not should_enqueue: - _clear_followup_pending(redis_client) - return - else: - _clear_bootstrap_failure_streak(redis_client) - try: - self.app.send_task( - "iaai_scraper.worker.tasks.sync_listing_task", - kwargs={ - "make": make, - "model": model, - "lane": lane, - "limit": limit, - "only_new": only_new, - }, - queue="scraping", - countdown=max(0, int(delay_seconds)), - ) - logger.info( - "Bootstrap follow-up sync queued in %ss (reason=%s)", - delay_seconds, - reason, - ) - except Exception: - _clear_followup_pending(redis_client) - logger.warning("Failed to enqueue bootstrap follow-up sync", exc_info=True) - - lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) - - if not lock_acquired: - orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id) - if orphan_cleared: - lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) - - if not lock_acquired: - logger.info("sync_listing_task skipped: another sync is already running") - return { - "status": "skipped", - "reason": "sync_already_running", - "task_id": task_id, - } - - try: - full_scan_done_before_run = _is_full_scan_done(redis_client) - force_bootstrap_full_scan = not full_scan_done_before_run - effective_limit = None if force_bootstrap_full_scan else limit - effective_only_new = False if force_bootstrap_full_scan else only_new - - # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. - # Используется только во время bootstrap для пропуска уже обработанных сегментов. - # Никаких page-level resume — внутри сегмента всегда стартуем с page 1. - last_completed_segment: int | None = None - if force_bootstrap_full_scan: - last_completed_segment = _load_last_completed_segment(redis_client) - else: - # После завершения bootstrap чекпоинт не нужен никогда. - _clear_sync_checkpoint(redis_client) - - if force_bootstrap_full_scan: - logger.info( - "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run", - ) - - logger.info( - "sync_listing options: only_new=%s, limit=%s", - effective_only_new, - effective_limit, - ) - - _clear_followup_pending(redis_client) - - heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( - redis_client, - SYNC_LISTING_LOCK_KEY, - owner_token, - lock_ttl, - ) - self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) - - # Определяем сегменты из конфига. - settings = Settings() - segments = parse_listing_segments(settings.listing.listing_segments_json) - use_segmented = bool(segments) and make is None and model is None - - resume_from_segment = 0 - if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None: - resume_from_segment = max(0, last_completed_segment + 1) - if resume_from_segment >= len(segments): - # Все сегменты уже пройдены — чекпоинт устарел, начинаем заново. - logger.info( - "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0", - last_completed_segment, len(segments), - ) - resume_from_segment = 0 - _clear_sync_checkpoint(redis_client) - elif resume_from_segment > 0: - logger.warning( - "Resuming segmented bootstrap from segment=%d (last completed=%d)", - resume_from_segment, last_completed_segment, - ) - - def _job(): - with IAAIScraper() as scraper: - if use_segmented: - return scraper.sync_listing_segmented( - segments=segments, - lane=lane, - only_new=effective_only_new, - start_segment=resume_from_segment, - start_page=1, - progress_callback=( - (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx)) - if force_bootstrap_full_scan else None - ), - ) - return scraper.sync_listing( - make=make, - model=model, - lane=lane, - limit=effective_limit, - only_new=effective_only_new, - ) - - result = _run_browser_job(_job) - - if force_bootstrap_full_scan: - bootstrap_completed = bool(result.get("full_scan_completed")) - if bootstrap_completed: - _set_full_scan_done(redis_client, True) - _clear_sync_checkpoint(redis_client) - _clear_bootstrap_failure_streak(redis_client) - logger.info("Bootstrap full scan completed; hourly schedule continues") - else: - _set_full_scan_done(redis_client, False) - listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {} - had_progress = any( - int(result.get(key) or 0) > 0 - for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered") - ) or int(listing_payload.get("vehicles_collected") or 0) > 0 - count_as_failure = str(result.get("status") or "") == "failed" and not had_progress - logger.info("Bootstrap full scan not complete yet; queuing immediate continuation") - _enqueue_bootstrap_followup( - "bootstrap_not_completed", - count_as_failure=count_as_failure, - ) - else: - _clear_sync_checkpoint(redis_client) - - summary = { - "task_id": task_id, - "run_id": result.get("run_id"), - "status": result.get("status", "success"), - "cars_upserted": result.get("cars_upserted", 0), - "cars_failed": result.get("cars_failed", 0), - "images_upserted": result.get("images_upserted", 0), - "skipped_existing": result.get("skipped_existing", 0), - "elapsed_seconds": result.get("elapsed_seconds"), - "failures_count": len(result.get("failures") or []), - } - logger.info( - "sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d", - summary["status"], - summary["cars_upserted"], - summary["cars_failed"], - summary["failures_count"], - ) - return summary - - except SoftTimeLimitExceeded: - logger.warning( - "sync_listing_task soft timeout exceeded — partial progress already saved to DB" - ) - if force_bootstrap_full_scan: - _set_full_scan_done(redis_client, False) - # SoftTimeLimit считаем failure для circuit breaker: - # если сегмент систематически не укладывается в time limit, - # нельзя крутить followup бесконечно. - _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=True) - # Partial progress уже записан в БД через finish_sync_run. - # Не retry — следующий запуск продолжит обработку по расписанию. - return { - "status": "timed_out", - "task_id": task_id, - "reason": "soft_time_limit_exceeded", - "note": "partial progress saved to DB; bootstrap continuation queued", - } - - except Exception as exc: - logger.error("sync_listing_task failed: %s", exc, exc_info=True) - # Retry только на не-таймаутные ошибки (сеть, БД, браузер). - try: - if force_bootstrap_full_scan: - _set_full_scan_done(redis_client, False) - raise self.retry(exc=exc, countdown=5) - raise self.retry(exc=exc) - except self.MaxRetriesExceededError: - logger.error("sync_listing_task max retries exceeded, giving up") - if force_bootstrap_full_scan: - _set_full_scan_done(redis_client, False) - _enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True) - return { - "status": "failed", - "task_id": task_id, - "error": str(exc), - } - finally: - if heartbeat_stop is not None: - heartbeat_stop.set() - if heartbeat_thread is not None: - heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) - if lock_acquired: - _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) diff --git a/iaai_scraper/__init__.py b/openlane_scraper/__init__.py similarity index 100% rename from iaai_scraper/__init__.py rename to openlane_scraper/__init__.py diff --git a/iaai_scraper/api/__init__.py b/openlane_scraper/api/__init__.py similarity index 100% rename from iaai_scraper/api/__init__.py rename to openlane_scraper/api/__init__.py diff --git a/iaai_scraper/api/app.py b/openlane_scraper/api/app.py similarity index 91% rename from iaai_scraper/api/app.py rename to openlane_scraper/api/app.py index ba73b00..ed66834 100644 --- a/iaai_scraper/api/app.py +++ b/openlane_scraper/api/app.py @@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI: _settings = settings or Settings() app = FastAPI( - title="IAAI Scraper API", - description="REST API для управления задачами скрапинга IAAI и просмотра данных", + title="OpenLane Scraper API", + description="REST API для управления задачами скрапинга OpenLane и просмотра данных", version="1.0.0", lifespan=lifespan, ) diff --git a/iaai_scraper/api/deps.py b/openlane_scraper/api/deps.py similarity index 100% rename from iaai_scraper/api/deps.py rename to openlane_scraper/api/deps.py diff --git a/iaai_scraper/api/routes/__init__.py b/openlane_scraper/api/routes/__init__.py similarity index 100% rename from iaai_scraper/api/routes/__init__.py rename to openlane_scraper/api/routes/__init__.py diff --git a/iaai_scraper/api/routes/cars.py b/openlane_scraper/api/routes/cars.py similarity index 100% rename from iaai_scraper/api/routes/cars.py rename to openlane_scraper/api/routes/cars.py diff --git a/iaai_scraper/api/routes/health.py b/openlane_scraper/api/routes/health.py similarity index 88% rename from iaai_scraper/api/routes/health.py rename to openlane_scraper/api/routes/health.py index a7de8d7..65273b3 100644 --- a/iaai_scraper/api/routes/health.py +++ b/openlane_scraper/api/routes/health.py @@ -9,7 +9,7 @@ from ..deps import get_persistence from ...storage.db import PersistenceService router = APIRouter() -logger = logging.getLogger("iaai_scraper.api.health") +logger = logging.getLogger("openlane_scraper.api.health") @router.get("/health") @@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)): return { "status": "ok" if db_ok else "degraded", - "service": "iaai-scraper-api", + "service": "openlane-scraper-api", "database": "connected" if db_ok else "unavailable", } diff --git a/iaai_scraper/api/routes/tasks.py b/openlane_scraper/api/routes/tasks.py similarity index 73% rename from iaai_scraper/api/routes/tasks.py rename to openlane_scraper/api/routes/tasks.py index 3979d34..e277ded 100644 --- a/iaai_scraper/api/routes/tasks.py +++ b/openlane_scraper/api/routes/tasks.py @@ -8,53 +8,30 @@ from ..deps import get_persistence from ...storage.db import PersistenceService from ...storage.models import SyncRun from ...worker.celery_app import celery_app -from ...worker.tasks import sync_vehicle_task, sync_listing_task +from ...worker.tasks import sync_listing_task router = APIRouter() -class SyncVehicleRequest(BaseModel): - vehicle_url: str - lane: str = "iaai" - - class SyncListingRequest(BaseModel): - make: str | None = None - model: str | None = None - lane: str = "iaai_cars" + lane: str = "openlane_marketplace" limit: int | None = None only_new: bool | None = None - - -@router.post("/tasks/sync-vehicle") -def start_sync_vehicle( - body: SyncVehicleRequest, -): - # Запустить задачу скрапинга одного автомобиля через Celery - result = sync_vehicle_task.apply_async( - kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, - queue="scraping", - ) - - return { - "task_id": result.id, - "status": "queued", - "vehicle_url": body.vehicle_url, - } + max_pages: int | None = None + concurrency: int | None = None @router.post("/tasks/sync-listing") def start_sync_listing( body: SyncListingRequest, ): - # Запустить задачу полного цикла листинга через Celery result = sync_listing_task.apply_async( kwargs={ - "make": body.make, - "model": body.model, "lane": body.lane, "limit": body.limit, "only_new": body.only_new, + "max_pages": body.max_pages, + "concurrency": body.concurrency, }, queue="scraping", ) @@ -90,7 +67,6 @@ def list_sync_runs( per_page: int = Query(20, ge=1, le=100), persistence: PersistenceService = Depends(get_persistence), ): - # История запусков синхронизации with persistence.session_scope() as session: total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 offset = (page - 1) * per_page diff --git a/openlane_scraper/browser/__init__.py b/openlane_scraper/browser/__init__.py new file mode 100644 index 0000000..9def84b --- /dev/null +++ b/openlane_scraper/browser/__init__.py @@ -0,0 +1,3 @@ +from .factory import BrowserFactory + +__all__ = ["BrowserFactory"] diff --git a/iaai_scraper/browser/factory.py b/openlane_scraper/browser/factory.py similarity index 89% rename from iaai_scraper/browser/factory.py rename to openlane_scraper/browser/factory.py index eb1f578..1a4954b 100644 --- a/iaai_scraper/browser/factory.py +++ b/openlane_scraper/browser/factory.py @@ -11,7 +11,7 @@ except ImportError: from ..core.config import Settings -logger = logging.getLogger("iaai_scraper.browser") +logger = logging.getLogger("openlane_scraper.browser") def _build_init_script() -> str: @@ -72,12 +72,12 @@ class BrowserFactory: # Выбор движка браузера. engine = self.settings.browser_engine.strip().lower() if engine == "auto": - # Для IAAI в headless-режиме Chromium со stealth-скриптами + # В headless-режиме Chromium со stealth-скриптами # значительно стабильнее Firefox по anti-bot. return "chromium" if engine in ("firefox", "chromium"): return engine - logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine) + logger.warning("Unknown OPENLANE_BROWSER_ENGINE=%r, falling back to auto", engine) return "chromium" def create_browser(self, playwright: Playwright) -> Browser: @@ -136,7 +136,7 @@ class BrowserFactory: logger.warning("Chrome channel launch failed, falling back to Chromium") return playwright.chromium.launch(**launch_kwargs) - def create_context(self, browser: Browser) -> BrowserContext: + def create_context(self, browser: Browser, storage_state_path: str | None = None) -> BrowserContext: viewport = random.choice(self.settings.fingerprint.viewport_presets) timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) color_scheme = random.choice(["light", "dark"]) @@ -178,6 +178,9 @@ class BrowserFactory: "Sec-CH-UA-Platform": '"Windows"', } + if storage_state_path: + ctx_kwargs["storage_state"] = storage_state_path + context = browser.new_context(**ctx_kwargs) context.set_default_timeout(self.settings.default_timeout_ms) context.set_default_navigation_timeout(self.settings.default_timeout_ms) @@ -193,12 +196,20 @@ class BrowserFactory: @staticmethod def enable_resource_blocking(page) -> None: - # Блокируем тяжёлые ресурсы для ускорения загрузки страниц. - BLOCKED_TYPES = {"image", "stylesheet", "font", "media"} + # Блокируем всё кроме document и XHR/fetch — минимальный след. + BLOCKED_TYPES = {"image", "stylesheet", "font", "media", "websocket", "eventsource", "manifest", "other"} BLOCKED_URL_PATTERNS = ( "google-analytics", "googletagmanager", "facebook.net", "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2", "analytics", "tracking", "adservice", + # OpenLane-трекеры из сетевого лога. + "bugsnag", "sessions.bugsnag", + "intercom", "widget.intercom", + "pusher", "sockjs", + "segment", "cdn.segment", + "ap3.com", "ap3c.com", + "onetrust", "optanon", "cookielaw", + "fullstory", "sentry", ) def _handle_route(route): diff --git a/openlane_scraper/cli.py b/openlane_scraper/cli.py new file mode 100644 index 0000000..d24897a --- /dev/null +++ b/openlane_scraper/cli.py @@ -0,0 +1,92 @@ +import argparse +from pathlib import Path + +from .core.config import Settings +from .core.utils import save_to_json +from .openlane import OpenLaneScrapeRunner +from .scraper import OpenLaneScraper + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="OpenLane scraper CLI") + parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") + parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("init-db", help="Create DB tables") + + sync_parser = subparsers.add_parser("sync-listing", help="Sync OpenLane marketplace → DB") + sync_parser.add_argument("--lane", default="openlane_marketplace") + sync_parser.add_argument("--limit", type=int, default=None) + sync_parser.add_argument("--only-new", choices=["true", "false"], default=None) + sync_parser.add_argument("--max-pages", type=int, default=None) + sync_parser.add_argument("--concurrency", type=int, default=None) + sync_parser.add_argument("--output", default=str(Path("artifacts/json/openlane_sync_listing.json"))) + + scrape_parser = subparsers.add_parser("scrape-openlane", help="Scrape OpenLane marketplace to JSONL files") + scrape_parser.add_argument("--max-pages", type=int, default=None) + scrape_parser.add_argument("--concurrency", type=int, default=None) + scrape_parser.add_argument("--resume", action="store_true") + scrape_parser.add_argument("--checkpoint-every-pages", type=int, default=None) + + subparsers.add_parser("openlane-login", help="Open interactive OpenLane login and persist browser storage state") + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + runtime_settings: Settings | None = None + if args.headless is not None or args.debug: + runtime_settings = Settings() + if args.headless is not None: + runtime_settings.headless = args.headless == "true" + if args.debug: + runtime_settings.log_level = "DEBUG" + + if args.command == "openlane-login": + runner = OpenLaneScrapeRunner(runtime_settings) + storage_state_path = runner.interactive_login() + print(f"OpenLane storage state saved to {Path(storage_state_path).resolve()}") + return + + if args.command == "scrape-openlane": + runner = OpenLaneScrapeRunner(runtime_settings) + result = runner.run( + max_pages=args.max_pages, + concurrency=args.concurrency, + resume=args.resume, + checkpoint_every_pages=args.checkpoint_every_pages, + ) + print(f"OpenLane JSONL saved to {Path(result.jsonl_path).resolve()}") + print(f"OpenLane aggregated JSON saved to {Path(result.aggregated_path).resolve()}") + print(f"OpenLane checkpoint saved to {Path(result.checkpoint_path).resolve()}") + print(f"OpenLane storage state saved to {Path(result.storage_state_path).resolve()}") + print( + f"OpenLane completed pages={len(result.completed_pages)} failed pages={len(result.failed_pages)} total_records={result.total_records} elapsed={result.elapsed_seconds:.2f}s" + ) + return + + with OpenLaneScraper(runtime_settings) as scraper: + if args.command == "init-db": + data = scraper.init_db() + print(f"DB initialized: {data}") + return + elif args.command == "sync-listing": + only_new = None if args.only_new is None else args.only_new == "true" + data = scraper.sync_listing( + lane=args.lane, + limit=args.limit, + only_new=only_new, + max_pages=args.max_pages, + concurrency=args.concurrency, + ) + + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + + +if __name__ == "__main__": + main() diff --git a/iaai_scraper/core/__init__.py b/openlane_scraper/core/__init__.py similarity index 100% rename from iaai_scraper/core/__init__.py rename to openlane_scraper/core/__init__.py diff --git a/openlane_scraper/core/config.py b/openlane_scraper/core/config.py new file mode 100644 index 0000000..fdac325 --- /dev/null +++ b/openlane_scraper/core/config.py @@ -0,0 +1,194 @@ +import os +from dataclasses import dataclass, field +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +# Хелперы для чтения env-переменных с приведением типов + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + return value if value is not None else default + + +def _env_optional_str(name: str) -> str | None: + value = os.getenv(name) + if value is None: + return None + value = value.strip() + return value or None + + +def _env_path_str(name: str) -> str | None: + value = _env_optional_str(name) + if value is None: + return None + return str(Path(value).expanduser()) + + +def _env_bool(name: str, default: bool) -> bool: + fallback = "true" if default else "false" + return _env_str(name, fallback).strip().lower() in TRUE_VALUES + + +def _env_int(name: str, default: int) -> int: + return int(_env_str(name, str(default)).strip()) + + +def _env_float(name: str, default: float) -> float: + return float(_env_str(name, str(default)).strip()) + + +# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +# --- Конфиг OpenLane (auth, paging, throttle, retry, checkpoint/output) --- + +@dataclass(slots=True) +class OpenLaneConfig: + sign_in_url: str = _env_str("OPENLANE_SIGN_IN_URL", "https://app.openlane.com/sign_in") + search_url: str = _env_str("OPENLANE_SEARCH_URL", "https://app.openlane.com/search?tab=marketplace") + api_search_url: str = _env_str("OPENLANE_API_SEARCH_URL", "https://app.openlane.com/api/v4/search") + username: str = _env_str("OPENLANE_USERNAME", "") + password: str = _env_str("OPENLANE_PASSWORD", "") + source_tab: str = _env_str("OPENLANE_SOURCE_TAB", "marketplace") + sale_types: str = _env_str("OPENLANE_SALE_TYPES", "marketplace") + page_size: int = _env_int("OPENLANE_PAGE_SIZE", 30) + max_pages: int = _env_int("OPENLANE_MAX_PAGES", 512) + concurrency: int = _env_int("OPENLANE_CONCURRENCY", 4) + throttle_min_seconds: float = _env_float("OPENLANE_THROTTLE_MIN_SECONDS", 0.3) + throttle_max_seconds: float = _env_float("OPENLANE_THROTTLE_MAX_SECONDS", 0.8) + retry_schedule_seconds: tuple[float, ...] = tuple( + float(part.strip()) + for part in _env_str("OPENLANE_RETRY_SCHEDULE_SECONDS", "2,5,10").split(",") + if part.strip() + ) or (2.0, 5.0, 10.0) + checkpoint_every_pages: int = _env_int("OPENLANE_CHECKPOINT_EVERY_PAGES", 10) + output_dir: str = _env_str("OPENLANE_OUTPUT_DIR", "artifacts/openlane") + jsonl_output: str = _env_str("OPENLANE_JSONL_OUTPUT", "artifacts/openlane/openlane_search.jsonl") + aggregated_output: str = _env_str("OPENLANE_AGGREGATED_OUTPUT", "artifacts/openlane/openlane_search_aggregated.json") + checkpoint_file: str = _env_str("OPENLANE_CHECKPOINT_FILE", "artifacts/openlane/openlane_checkpoint.json") + storage_state_file: str = _env_str("OPENLANE_STORAGE_STATE_FILE", "artifacts/openlane/openlane_storage_state.json") + persist_storage_state: bool = _env_bool("OPENLANE_PERSIST_STORAGE_STATE", True) + progress_log_every_pages: int = _env_int("OPENLANE_PROGRESS_LOG_EVERY_PAGES", 10) + request_timeout_ms: int = _env_int("OPENLANE_REQUEST_TIMEOUT_MS", 45000) + max_cars_limit: int = _env_int("OPENLANE_MAX_CARS_LIMIT", 20) + refresh_token: str = _env_str("OPENLANE_REFRESH_TOKEN", "") + okta_client_id: str = _env_str("OPENLANE_OKTA_CLIENT_ID", "") + okta_token_endpoint: str = _env_str( + "OPENLANE_OKTA_TOKEN_ENDPOINT", + "https://okta.iam.karglobal.com/oauth2/default/v1/token", + ) + okta_redirect_uri: str = _env_str("OPENLANE_OKTA_REDIRECT_URI", "https://app.openlane.com/sign_in") + okta_timeout_seconds: int = _env_int("OPENLANE_OKTA_TIMEOUT_SECONDS", 15) + + +# --- Конфиг PostgreSQL (URL, пул соединений, pool_recycle) --- + +@dataclass(slots=True) +class DatabaseConfig: + url: str = _env_str("OPENLANE_DATABASE_URL", "postgresql+psycopg2://openlane:openlane@localhost:5432/openlane_scraper") + echo: bool = _env_bool("OPENLANE_DATABASE_ECHO", False) + pool_size: int = _env_int("OPENLANE_DATABASE_POOL_SIZE", 5) + max_overflow: int = _env_int("OPENLANE_DATABASE_MAX_OVERFLOW", 10) + pool_recycle_seconds: int = _env_int("OPENLANE_DATABASE_POOL_RECYCLE_SECONDS", 1800) + auto_create_tables: bool = _env_bool("OPENLANE_DATABASE_AUTO_CREATE_TABLES", False) + + +# --- Конфиг Redis (URL для Celery broker) --- + +@dataclass(slots=True) +class RedisConfig: + url: str = _env_str("OPENLANE_REDIS_URL", "redis://localhost:6379/0") + socket_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) + socket_connect_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) + health_check_interval_seconds: int = _env_int("OPENLANE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) + + +# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = _env_str("CELERY_BROKER_URL", "") + result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") + task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) + task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) + worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) + worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) + broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) + beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) + beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None + + +# --- Конфиг прокси (server, username, password) --- + +@dataclass(slots=True) +class ProxyConfig: + server: str | None = _env_optional_str("OPENLANE_PROXY_SERVER") + username: str | None = _env_optional_str("OPENLANE_PROXY_USERNAME") + password: str | None = _env_optional_str("OPENLANE_PROXY_PASSWORD") + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + +# Главный объект настроек: собирает все блоки конфигурации + +@dataclass(slots=True) +class Settings: + default_timeout_ms: int = _env_int("OPENLANE_TIMEOUT_MS", 45000) + headless: bool = _env_bool("OPENLANE_HEADLESS", True) + browser_engine: str = _env_str("OPENLANE_BROWSER_ENGINE", "auto") + log_level: str = _env_str("OPENLANE_LOG_LEVEL", "INFO") + log_file: str | None = _env_optional_str("OPENLANE_LOG_FILE") + enable_trace_id_logs: bool = _env_bool("OPENLANE_ENABLE_TRACE_ID_LOGS", True) + runtime_config_file: str | None = _env_path_str("OPENLANE_RUNTIME_CONFIG_FILE") + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + openlane: OpenLaneConfig = field(default_factory=OpenLaneConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) + + +# Глобальный синглтон — используется по умолчанию во всех модулях. +settings = Settings() diff --git a/iaai_scraper/core/logs.py b/openlane_scraper/core/logs.py similarity index 100% rename from iaai_scraper/core/logs.py rename to openlane_scraper/core/logs.py diff --git a/openlane_scraper/core/runtime_config.py b/openlane_scraper/core/runtime_config.py new file mode 100644 index 0000000..afe6f84 --- /dev/null +++ b/openlane_scraper/core/runtime_config.py @@ -0,0 +1,379 @@ +"""Runtime config: загрузка и применение runtime_config.json. + +Позволяет менять параметры sync и фильтры машин без перезапуска сервиса. +Файл перечитывается при каждом запуске sync_listing. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger("openlane_scraper.core.runtime_config") + + +@dataclass(slots=True) +class SyncConfig: + name: str | None = None + ids_initial_size: int | None = None + ids_next_size: int | None = None + ids_max_pages: int | None = None + condition_check_enabled: bool | None = None + lane: str = "openlane_marketplace" + only_new: bool = False + limit: int | None = None + + +@dataclass(slots=True) +class ListingConfig: + make: str | None = None + model: str | None = None + + +@dataclass(slots=True) +class RangeFilter: + min: int | None = None + max: int | None = None + + +@dataclass(slots=True) +class FlagFilters: + damaged_only: bool | None = None + run_and_drive: bool | None = None + + +@dataclass(slots=True) +class FieldFilters: + brands: list[str] = field(default_factory=list) + models: list[str] = field(default_factory=list) + years: list[int] = field(default_factory=list) + body_types: list[str] = field(default_factory=list) + colors: list[str] = field(default_factory=list) + drives: list[str] = field(default_factory=list) + gearboxes: list[str] = field(default_factory=list) + locations: list[str] = field(default_factory=list) + + +@dataclass(slots=True) +class Filters: + price: RangeFilter = field(default_factory=RangeFilter) + mileage: RangeFilter = field(default_factory=RangeFilter) + flags: FlagFilters = field(default_factory=FlagFilters) + + include: FieldFilters = field(default_factory=FieldFilters) + exclude: FieldFilters = field(default_factory=FieldFilters) + + # Legacy flat fields (backward compatibility). + brands: list[str] = field(default_factory=list) + models: list[str] = field(default_factory=list) + years: list[int] = field(default_factory=list) + body_types: list[str] = field(default_factory=list) + colors: list[str] = field(default_factory=list) + drives: list[str] = field(default_factory=list) + gearboxes: list[str] = field(default_factory=list) + locations: list[str] = field(default_factory=list) + + exclude_brands: list[str] = field(default_factory=list) + exclude_models: list[str] = field(default_factory=list) + exclude_years: list[int] = field(default_factory=list) + exclude_body_types: list[str] = field(default_factory=list) + exclude_colors: list[str] = field(default_factory=list) + exclude_drives: list[str] = field(default_factory=list) + exclude_gearboxes: list[str] = field(default_factory=list) + exclude_locations: list[str] = field(default_factory=list) + + +@dataclass(slots=True) +class RuntimeConfig: + sync: SyncConfig = field(default_factory=SyncConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + filters: Filters = field(default_factory=Filters) + + +def load_runtime_config(path: str | Path | None) -> RuntimeConfig: + """Загрузить runtime_config.json. Если файл отсутствует — вернуть дефолты.""" + if not path: + return RuntimeConfig() + + p = Path(path) + if not p.exists(): + logger.debug("runtime_config not found at %s — using defaults", p) + return RuntimeConfig() + + try: + raw = json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + logger.warning("Failed to parse runtime_config %s: %s — using defaults", p, exc) + return RuntimeConfig() + + cfg = RuntimeConfig() + + sync_raw = raw.get("sync") or {} + if isinstance(sync_raw, dict): + cfg.sync.name = _opt_str(sync_raw.get("name")) + cfg.sync.ids_initial_size = _opt_int(sync_raw.get("ids_initial_size")) + cfg.sync.ids_next_size = _opt_int(sync_raw.get("ids_next_size")) + cfg.sync.ids_max_pages = _opt_int(sync_raw.get("ids_max_pages")) + cfg.sync.condition_check_enabled = _opt_bool(sync_raw.get("condition_check_enabled")) + cfg.sync.lane = _opt_str(sync_raw.get("lane")) or cfg.sync.lane + only_new = _opt_bool(sync_raw.get("only_new")) + if only_new is not None: + cfg.sync.only_new = only_new + cfg.sync.limit = _opt_int(sync_raw.get("limit")) + + listing_raw = raw.get("listing") or {} + if isinstance(listing_raw, dict): + cfg.listing.make = _opt_str(listing_raw.get("make")) + cfg.listing.model = _opt_str(listing_raw.get("model")) + + filters_raw = raw.get("filters") or {} + if isinstance(filters_raw, dict): + cfg.filters.price = _parse_range(filters_raw.get("price")) + cfg.filters.mileage = _parse_range(filters_raw.get("mileage")) + cfg.filters.flags = _parse_flags(filters_raw.get("flags")) + + include_payload = filters_raw.get("include") if isinstance(filters_raw.get("include"), dict) else filters_raw + exclude_payload = ( + filters_raw.get("exclude") + if isinstance(filters_raw.get("exclude"), dict) + else { + "brands": filters_raw.get("exclude_brands"), + "models": filters_raw.get("exclude_models"), + "years": filters_raw.get("exclude_years"), + "body_types": filters_raw.get("exclude_body_types"), + "colors": filters_raw.get("exclude_colors"), + "drives": filters_raw.get("exclude_drives"), + "gearboxes": filters_raw.get("exclude_gearboxes"), + "locations": filters_raw.get("exclude_locations"), + } + ) + + cfg.filters.include = _parse_fields(include_payload) + cfg.filters.exclude = _parse_fields(exclude_payload) + + # Backward-compatible flat aliases. + cfg.filters.brands = list(cfg.filters.include.brands) + cfg.filters.models = list(cfg.filters.include.models) + cfg.filters.years = list(cfg.filters.include.years) + cfg.filters.body_types = list(cfg.filters.include.body_types) + cfg.filters.colors = list(cfg.filters.include.colors) + cfg.filters.drives = list(cfg.filters.include.drives) + cfg.filters.gearboxes = list(cfg.filters.include.gearboxes) + cfg.filters.locations = list(cfg.filters.include.locations) + + cfg.filters.exclude_brands = list(cfg.filters.exclude.brands) + cfg.filters.exclude_models = list(cfg.filters.exclude.models) + cfg.filters.exclude_years = list(cfg.filters.exclude.years) + cfg.filters.exclude_body_types = list(cfg.filters.exclude.body_types) + cfg.filters.exclude_colors = list(cfg.filters.exclude.colors) + cfg.filters.exclude_drives = list(cfg.filters.exclude.drives) + cfg.filters.exclude_gearboxes = list(cfg.filters.exclude.gearboxes) + cfg.filters.exclude_locations = list(cfg.filters.exclude.locations) + + logger.info( + "Loaded runtime_config from %s: lane=%s, limit=%s", + p, + cfg.sync.lane, + cfg.sync.limit, + ) + return cfg + + +def apply_filters(records: list[dict], filters: Filters) -> list[dict]: + """Применить runtime-фильтры к списку сырых записей из API.""" + if not records: + return records + + include = getattr(filters, "include", None) + exclude = getattr(filters, "exclude", None) + + include_brands = tuple(getattr(filters, "brands", ()) or getattr(include, "brands", ())) + include_models = tuple(getattr(filters, "models", ()) or getattr(include, "models", ())) + include_years = tuple(getattr(filters, "years", ()) or getattr(include, "years", ())) + include_body_types = tuple(getattr(filters, "body_types", ()) or getattr(include, "body_types", ())) + + exclude_brands = tuple(getattr(filters, "exclude_brands", ()) or getattr(exclude, "brands", ())) + exclude_models = tuple(getattr(filters, "exclude_models", ()) or getattr(exclude, "models", ())) + exclude_years = tuple(getattr(filters, "exclude_years", ()) or getattr(exclude, "years", ())) + exclude_body_types = tuple(getattr(filters, "exclude_body_types", ()) or getattr(exclude, "body_types", ())) + + price_cfg = getattr(filters, "price", None) + mileage_cfg = getattr(filters, "mileage", None) + flags_cfg = getattr(filters, "flags", None) + + price_min = getattr(price_cfg, "min", None) + price_max = getattr(price_cfg, "max", None) + mileage_min = getattr(mileage_cfg, "min", None) + mileage_max = getattr(mileage_cfg, "max", None) + damaged_only = getattr(flags_cfg, "damaged_only", None) + run_and_drive = getattr(flags_cfg, "run_and_drive", None) + + result = records + + if include_brands: + include_set = {s.casefold() for s in include_brands} + result = [r for r in result if _get_brand(r).casefold() in include_set] + if exclude_brands: + exclude_set = {s.casefold() for s in exclude_brands} + result = [r for r in result if _get_brand(r).casefold() not in exclude_set] + + if include_models: + include_set = {s.casefold() for s in include_models} + result = [r for r in result if _get_model(r).casefold() in include_set] + if exclude_models: + exclude_set = {s.casefold() for s in exclude_models} + result = [r for r in result if _get_model(r).casefold() not in exclude_set] + + if include_years: + years_set = set(include_years) + result = [r for r in result if _get_year(r) in years_set] + if exclude_years: + years_set = set(exclude_years) + result = [r for r in result if _get_year(r) not in years_set] + + if include_body_types: + include_set = {s.casefold() for s in include_body_types} + result = [r for r in result if _get_body_type(r).casefold() in include_set] + if exclude_body_types: + exclude_set = {s.casefold() for s in exclude_body_types} + result = [r for r in result if _get_body_type(r).casefold() not in exclude_set] + + if price_min is not None or price_max is not None: + filtered: list[dict] = [] + for r in result: + price = _get_price(r) + if price is None: + filtered.append(r) + continue + if price_min is not None and price < price_min: + continue + if price_max is not None and price > price_max: + continue + filtered.append(r) + result = filtered + + if mileage_min is not None or mileage_max is not None: + filtered = [] + for r in result: + miles = _get_mileage(r) + if miles is None: + filtered.append(r) + continue + if mileage_min is not None and miles < mileage_min: + continue + if mileage_max is not None and miles > mileage_max: + continue + filtered.append(r) + result = filtered + + if damaged_only is not None: + result = [r for r in result if bool(r.get("is_damaged")) is damaged_only] + if run_and_drive is not None: + result = [r for r in result if bool(r.get("run_and_drive")) is run_and_drive] + + return result + + +def _parse_range(payload: Any) -> RangeFilter: + payload = payload if isinstance(payload, dict) else {} + return RangeFilter(min=_opt_int(payload.get("min")), max=_opt_int(payload.get("max"))) + + +def _parse_flags(payload: Any) -> FlagFilters: + payload = payload if isinstance(payload, dict) else {} + return FlagFilters( + damaged_only=_opt_bool(payload.get("damaged_only")), + run_and_drive=_opt_bool(payload.get("run_and_drive")), + ) + + +def _parse_fields(payload: Any) -> FieldFilters: + payload = payload if isinstance(payload, dict) else {} + return FieldFilters( + brands=_str_list(payload.get("brands")), + models=_str_list(payload.get("models")), + years=_int_list(payload.get("years")), + body_types=_str_list(payload.get("body_types")), + colors=_str_list(payload.get("colors")), + drives=_str_list(payload.get("drives")), + gearboxes=_str_list(payload.get("gearboxes")), + locations=_str_list(payload.get("locations")), + ) + + +def _opt_int(value: Any) -> int | None: + if value is None: + return None + try: + return int(value) + except (ValueError, TypeError): + return None + + +def _opt_bool(value: Any) -> bool | None: + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().casefold() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return None + + +def _opt_str(value: Any) -> str | None: + if value is None: + return None + s = str(value).strip() + return s or None + + +def _str_list(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + return [str(v).strip() for v in value if str(v).strip()] + + +def _int_list(value: Any) -> list[int]: + if not isinstance(value, list): + return [] + result: list[int] = [] + for v in value: + iv = _opt_int(v) + if iv is not None: + result.append(iv) + return result + + +def _get_brand(record: dict[str, Any]) -> str: + return str(record.get("make") or record.get("brand") or "") + + +def _get_model(record: dict[str, Any]) -> str: + return str(record.get("model") or "") + + +def _get_body_type(record: dict[str, Any]) -> str: + return str(record.get("body_type") or record.get("body_style") or record.get("vehicle_type") or "") + + +def _get_year(record: dict[str, Any]) -> int | None: + return _opt_int(record.get("year")) + + +def _get_price(record: dict[str, Any]) -> int | None: + for key in ("current_high_bid", "buy_now_price", "price", "current_bid", "sale_price"): + value = _opt_int(record.get(key)) + if value is not None: + return value + return None + + +def _get_mileage(record: dict[str, Any]) -> int | None: + return _opt_int(record.get("odometer") or record.get("mileage")) diff --git a/iaai_scraper/core/utils.py b/openlane_scraper/core/utils.py similarity index 93% rename from iaai_scraper/core/utils.py rename to openlane_scraper/core/utils.py index 98b00a8..6260d6f 100644 --- a/iaai_scraper/core/utils.py +++ b/openlane_scraper/core/utils.py @@ -17,11 +17,6 @@ def first_non_empty(values: Iterable[Any]) -> Any | None: return None -# Регулярные выражения для lot и price. -LOT_RE = re.compile(r"\b(\d{7,10})\b") -PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") - - def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве. found = [] diff --git a/openlane_scraper/openlane/__init__.py b/openlane_scraper/openlane/__init__.py new file mode 100644 index 0000000..b2597fc --- /dev/null +++ b/openlane_scraper/openlane/__init__.py @@ -0,0 +1,4 @@ +from .mapper import map_openlane_record, map_openlane_records +from .runner import OpenLaneScrapeRunner, OpenLaneScrapeResult + +__all__ = ["OpenLaneScrapeRunner", "OpenLaneScrapeResult", "map_openlane_record", "map_openlane_records"] diff --git a/openlane_scraper/openlane/auth.py b/openlane_scraper/openlane/auth.py new file mode 100644 index 0000000..3526c28 --- /dev/null +++ b/openlane_scraper/openlane/auth.py @@ -0,0 +1,484 @@ +from __future__ import annotations + +import base64 +import json +import logging +import os +import re +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +from playwright.sync_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError + +from ..browser.factory import BrowserFactory +from ..core.config import OpenLaneConfig + +logger = logging.getLogger("openlane_scraper.openlane.auth") + +# Порог предупреждения об истечении refresh_token (дни). +_TOKEN_REFRESH_THRESHOLD_DAYS = 3 +_ACCESS_TOKEN_MIN_TTL_SECONDS = 300 + + +class OpenLaneAuthError(RuntimeError): + pass + + +class OpenLaneAuthenticator: + def __init__(self, config: OpenLaneConfig) -> None: + self.config = config + + def validate_credentials(self) -> None: + if not self.config.username or not self.config.password: + raise OpenLaneAuthError( + "OPENLANE_USERNAME and OPENLANE_PASSWORD must be set in the environment" + ) + + def interactive_login_and_persist(self, context: BrowserContext) -> str: + page = context.new_page() + page.set_default_timeout(self.config.request_timeout_ms) + logger.info("OpenLane interactive login: open %s and complete sign-in manually", self.config.sign_in_url) + page.goto(self.config.sign_in_url, wait_until="domcontentloaded") + self._wait_for_authenticated_session(page) + self._persist_storage_state(context) + logger.info("OpenLane interactive login successful") + return str(Path(self.config.storage_state_file)) + + def _inject_refresh_token(self, context: BrowserContext) -> None: + # Инъекция refresh_token cookie из env. + token = self.config.refresh_token + if not token: + return + + # Проверка: токен уже истёк? + days = self._token_days_remaining(token) + if days is not None and days <= 0: + logger.warning( + "OPENLANE_REFRESH_TOKEN is EXPIRED (%.1f days ago). " + "Will attempt login with username/password.", + abs(days), + ) + # Очищаем токен — fallback на login() + self.config.refresh_token = "" + return + + if days is not None: + logger.info("refresh_token: %.1f days remaining (expires %s)", + days, datetime.fromtimestamp( + self._decode_jwt_exp(token), tz=timezone.utc # type: ignore[arg-type] + ).strftime("%Y-%m-%d %H:%M UTC")) + if days < _TOKEN_REFRESH_THRESHOLD_DAYS: + logger.warning( + "refresh_token expires in %.1f days! " + "Will try to re-login after auth to get a fresh one.", + days, + ) + + context.add_cookies([ + { + "name": "refresh_token", + "value": token, + "domain": ".openlane.com", + "path": "/", + "httpOnly": True, + "secure": True, + "sameSite": "None", + } + ]) + logger.info("Injected OPENLANE_REFRESH_TOKEN cookie into browser context") + + # ------------------------------------------------------------------ + # Инспекция и ротация токенов + # ------------------------------------------------------------------ + + @staticmethod + def _decode_jwt_exp(token: str) -> int | None: + # Декодирование JWT exp (без проверки подписи). + try: + parts = token.split(".") + if len(parts) < 2: + return None + payload_b64 = parts[1] + # Дополнение base64 padding. + payload_b64 += "=" * (-len(payload_b64) % 4) + payload = json.loads(base64.urlsafe_b64decode(payload_b64)) + exp = payload.get("exp") + return int(exp) if exp is not None else None + except Exception: + return None + + @staticmethod + def _decode_jwt_payload(token: str) -> dict[str, Any]: + # Декодирование JWT payload (без проверки подписи). + try: + parts = token.split(".") + if len(parts) < 2: + return {} + payload_b64 = parts[1] + payload_b64 += "=" * (-len(payload_b64) % 4) + payload = json.loads(base64.urlsafe_b64decode(payload_b64)) + return payload if isinstance(payload, dict) else {} + except Exception: + return {} + + @staticmethod + def _token_days_remaining(token: str) -> float | None: + # Дней до истечения токена. + exp = OpenLaneAuthenticator._decode_jwt_exp(token) + if exp is None: + return None + now = datetime.now(timezone.utc).timestamp() + return (exp - now) / 86400 + + @staticmethod + def _token_seconds_remaining(token: str) -> int | None: + # Секунд до истечения токена. + exp = OpenLaneAuthenticator._decode_jwt_exp(token) + if exp is None: + return None + return int(exp - time.time()) + + @staticmethod + def _extract_cookie_value(context: BrowserContext, name: str) -> str | None: + # Извлекаем значение cookie по имени. + cookies = context.cookies("https://app.openlane.com") + for cookie in cookies: + if cookie.get("name") == name: + value = str(cookie.get("value") or "").strip() + if value: + return value + return None + + def _is_access_token_fresh(self, token: str) -> bool: + # Проверяем что access_token ещё жив. + remain = self._token_seconds_remaining(token) + return remain is not None and remain > _ACCESS_TOKEN_MIN_TTL_SECONDS + + def _extract_refresh_token_from_cookies(self, context: BrowserContext) -> str | None: + # Извлечение refresh_token из cookies. + return self._extract_cookie_value(context, "refresh_token") + + def _extract_access_token_from_cookies(self, context: BrowserContext) -> str | None: + # Извлечение access_token из cookies. + return self._extract_cookie_value(context, "access_token") + + def _refresh_access_token_via_okta(self, refresh_token: str) -> tuple[str | None, str | None]: + # Прямой refresh через Okta token endpoint. + if not refresh_token: + return None, None + if not self.config.okta_client_id: + logger.warning("OPENLANE_OKTA_CLIENT_ID is empty, direct refresh disabled") + return None, None + + form_data = urlencode( + { + "grant_type": "refresh_token", + "client_id": self.config.okta_client_id, + "redirect_uri": self.config.okta_redirect_uri, + "refresh_token": refresh_token, + } + ).encode("utf-8") + req = Request( + self.config.okta_token_endpoint, + data=form_data, + method="POST", + headers={ + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json", + }, + ) + + try: + with urlopen(req, timeout=self.config.okta_timeout_seconds) as resp: + payload = json.loads(resp.read().decode("utf-8")) + access_token = str(payload.get("access_token") or "").strip() + new_refresh_token = str(payload.get("refresh_token") or "").strip() or refresh_token + if not access_token: + logger.warning("Okta refresh response has no access_token") + return None, None + logger.info("Okta direct refresh succeeded") + return access_token, new_refresh_token + except HTTPError as exc: + try: + body = exc.read().decode("utf-8", errors="ignore")[:300] + except Exception: + body = "" + logger.warning("Okta direct refresh failed: HTTP %s %s", exc.code, body) + return None, None + except URLError as exc: + logger.warning("Okta direct refresh failed: %s", exc) + return None, None + except Exception as exc: + logger.warning("Okta direct refresh failed: %s", exc) + return None, None + + def _persist_access_token_to_storage_state(self, context: BrowserContext, access_token: str) -> None: + # Сохраняем access_token cookie в контекст. + expires = self._decode_jwt_exp(access_token) + if expires is None: + expires = int(time.time()) + 7200 + context.add_cookies( + [ + { + "name": "access_token", + "value": access_token, + "domain": ".openlane.com", + "path": "/", + "httpOnly": True, + "secure": True, + "sameSite": "None", + "expires": int(expires), + } + ] + ) + + def _maybe_rotate_token(self, context: BrowserContext) -> None: + # Проверка и ротация refresh_token. + new_token = self._extract_refresh_token_from_cookies(context) + if not new_token: + logger.debug("No refresh_token cookie found after auth — nothing to rotate") + return + + old_token = self.config.refresh_token or "" + + # Проверяем изменился ли токен. + if new_token != old_token: + days = self._token_days_remaining(new_token) + logger.info( + "refresh_token CHANGED (new exp: %.1f days). Persisting to .env", + days if days is not None else -1, + ) + self.config.refresh_token = new_token + self._persist_token_to_dotenv(new_token) + return + + # Токен не изменился — проверяем срок. + days = self._token_days_remaining(new_token) + if days is not None: + logger.info("refresh_token unchanged, %.1f days remaining", days) + if days < _TOKEN_REFRESH_THRESHOLD_DAYS: + logger.warning( + "⚠ refresh_token expires in %.1f days! " + "Run `openlane-login` or set OPENLANE_USERNAME + OPENLANE_PASSWORD " + "to auto-renew on next sync.", + days, + ) + + @staticmethod + def _persist_token_to_dotenv(token: str) -> None: + # Запись refresh_token в .env. + env_path = Path(".env") + if not env_path.exists(): + # Создаём .env с токеном. + env_path.write_text( + f"OPENLANE_REFRESH_TOKEN={token}\n", + encoding="utf-8", + ) + logger.info("Created .env with rotated OPENLANE_REFRESH_TOKEN") + return + + content = env_path.read_text(encoding="utf-8") + pattern = re.compile(r"^OPENLANE_REFRESH_TOKEN=.*$", re.MULTILINE) + replacement = f"OPENLANE_REFRESH_TOKEN={token}" + + if pattern.search(content): + new_content = pattern.sub(replacement, content) + else: + new_content = content.rstrip("\n") + f"\n{replacement}\n" + + env_path.write_text(new_content, encoding="utf-8") + # Обновляем env процесса. + os.environ["OPENLANE_REFRESH_TOKEN"] = token + logger.info("Persisted rotated OPENLANE_REFRESH_TOKEN to .env") + + def bootstrap_authenticated_context(self, context: BrowserContext) -> Page: + # Восстановление сессии: storage_state → login → ошибка. + storage_state_path = Path(self.config.storage_state_file) + + if storage_state_path.exists(): + logger.info("OpenLane: restoring session from %s", storage_state_path) + page = context.new_page() + page.set_default_timeout(self.config.request_timeout_ms) + # Блокируем трекинг/картинки — нужен только fetch(). + BrowserFactory.enable_resource_blocking(page) + + # Проверка cookies без сетевых запросов. + cookies = context.cookies("https://app.openlane.com") + cookie_names = {c["name"] for c in cookies} + access_token = self._extract_access_token_from_cookies(context) + if access_token and self._is_access_token_fresh(access_token): + logger.info( + "OpenLane session restored: access_token cookie present (%d cookies total)", + len(cookies), + ) + # Лёгкий API endpoint для установки origin в браузере. + page.goto( + "https://app.openlane.com/api/_next/time", + wait_until="domcontentloaded", + ) + return page + + # Пробуем прямой refresh access_token без навигации. + refresh_token = self._extract_refresh_token_from_cookies(context) or self.config.refresh_token + if refresh_token: + new_access_token, new_refresh_token = self._refresh_access_token_via_okta(refresh_token) + if new_access_token: + self._persist_access_token_to_storage_state(context, new_access_token) + if new_refresh_token and new_refresh_token != (self.config.refresh_token or ""): + self.config.refresh_token = new_refresh_token + self._persist_token_to_dotenv(new_refresh_token) + self._persist_storage_state(context) + page.goto( + "https://app.openlane.com/api/_next/time", + wait_until="domcontentloaded", + ) + logger.info("OpenLane session restored via direct Okta refresh") + return page + logger.warning("Direct refresh failed, trying sign_in fallback") + + # Если refresh_token был только в env, добавляем cookie вручную. + if "refresh_token" not in cookie_names and self.config.refresh_token: + self._inject_refresh_token(context) + cookies = context.cookies("https://app.openlane.com") + cookie_names = {c["name"] for c in cookies} + + logger.warning("storage_state exists but no access_token cookie — trying navigation") + # Fallback: SPA обменяет refresh_token на access_token. + if "refresh_token" in cookie_names: + try: + page.goto(self.config.sign_in_url, wait_until="domcontentloaded") + self._wait_for_authenticated_session(page) + self._persist_storage_state(context) + self._maybe_rotate_token(context) + return page + except OpenLaneAuthError: + logger.warning("SPA refresh_token exchange failed") + page.close() + + # Нет storage_state — пробуем логин. + if self.config.username and self.config.password: + return self.login(context) + + raise OpenLaneAuthError( + "No valid session found. Either:\n" + " 1. Run `python scripts/explore_site.py` to login manually and save storage_state, or\n" + " 2. Set OPENLANE_USERNAME and OPENLANE_PASSWORD in .env" + ) + + def login(self, context: BrowserContext) -> Page: + self.validate_credentials() + page = context.new_page() + page.set_default_timeout(self.config.request_timeout_ms) + logger.info("OpenLane login: opening sign-in page %s", self.config.sign_in_url) + page.goto(self.config.sign_in_url, wait_until="domcontentloaded") + self._fill_login_form(page) + self._wait_for_authenticated_session(page) + self._persist_storage_state(context) + self._maybe_rotate_token(context) + logger.info("OpenLane login successful") + return page + + def _fill_login_form(self, page: Page) -> None: + email_selectors = [ + 'input[name="username"]', + 'input[name="user[login]"]', + 'input[name="email"]', + 'input[name="user[email]"]', + 'input[autocomplete="username"]', + 'input[type="email"]', + 'input[type="text"]', + '#email', + ] + password_selectors = [ + 'input[name="password"]', + 'input[name="user[password]"]', + 'input[type="password"]', + '#password', + ] + submit_selectors = [ + 'button[type="submit"]', + 'input[type="submit"]', + 'button:has-text("Sign in")', + 'button:has-text("Log in")', + ] + + email_filled = self._fill_first(page, email_selectors, self.config.username) + password_filled = self._fill_first(page, password_selectors, self.config.password) + if not email_filled or not password_filled: + raise OpenLaneAuthError("OpenLane login form fields were not found") + + if not self._click_first(page, submit_selectors): + raise OpenLaneAuthError("OpenLane login submit button was not found") + + def _wait_for_authenticated_session(self, page: Page) -> None: + # Ждём пока SPA обменяет refresh_token → access_token. + try: + page.wait_for_url( + lambda url: "/sign_in" not in url.lower(), + timeout=min(self.config.request_timeout_ms, 30000), + ) + except PlaywrightTimeoutError: + raise OpenLaneAuthError( + "OpenLane authentication failed: page stayed on sign_in " + "(refresh_token likely invalid or expired)" + ) + + # Проверяем что не на странице ошибки. + current_url = page.url.lower() + logger.debug("After auth redirect, URL: %s", page.url) + if "/sign_in" in current_url: + raise OpenLaneAuthError("OpenLane authentication failed: still on sign_in page") + + # Ждём появления access_token cookie. + for attempt in range(15): + cookies = page.context.cookies("https://app.openlane.com") + cookie_names = {c["name"] for c in cookies} + if "access_token" in cookie_names: + logger.info("OpenLane authenticated: access_token cookie present (attempt %d)", attempt) + return + page.wait_for_timeout(500) + + # Нет access_token cookie, но URL не sign_in — продолжаем. + logger.warning( + "access_token cookie not found after redirect, but page is at %s — continuing", + page.url, + ) + + def _persist_storage_state(self, context: BrowserContext) -> None: + if not self.config.persist_storage_state: + return + storage_state_path = Path(self.config.storage_state_file) + storage_state_path.parent.mkdir(parents=True, exist_ok=True) + context.storage_state(path=str(storage_state_path)) + logger.info("OpenLane storage state saved to %s", storage_state_path) + + @staticmethod + def _fill_first(page: Page, selectors: list[str], value: str) -> bool: + for selector in selectors: + locator = page.locator(selector) + if locator.count() == 0: + continue + try: + locator.first.fill(value) + return True + except PlaywrightTimeoutError: + continue + return False + + @staticmethod + def _click_first(page: Page, selectors: list[str]) -> bool: + for selector in selectors: + locator = page.locator(selector) + if locator.count() == 0: + continue + try: + locator.first.click() + return True + except PlaywrightTimeoutError: + continue + return False diff --git a/openlane_scraper/openlane/checkpoint.py b/openlane_scraper/openlane/checkpoint.py new file mode 100644 index 0000000..f9cb40c --- /dev/null +++ b/openlane_scraper/openlane/checkpoint.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field +from pathlib import Path + + +@dataclass(slots=True) +class OpenLaneCheckpoint: + max_pages: int + completed_pages: list[int] = field(default_factory=list) + failed_pages: list[int] = field(default_factory=list) + total_records: int = 0 + last_saved_at: str | None = None + + @property + def completed_set(self) -> set[int]: + return set(self.completed_pages) + + +class OpenLaneCheckpointStore: + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + + def load(self, max_pages: int) -> OpenLaneCheckpoint: + if not self.path.exists(): + return OpenLaneCheckpoint(max_pages=max_pages) + data = json.loads(self.path.read_text(encoding="utf-8")) + return OpenLaneCheckpoint( + max_pages=int(data.get("max_pages") or max_pages), + completed_pages=sorted({int(page) for page in data.get("completed_pages", [])}), + failed_pages=sorted({int(page) for page in data.get("failed_pages", [])}), + total_records=int(data.get("total_records") or 0), + last_saved_at=data.get("last_saved_at"), + ) + + def save(self, checkpoint: OpenLaneCheckpoint) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + payload = asdict(checkpoint) + self.path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) diff --git a/openlane_scraper/openlane/client.py b/openlane_scraper/openlane/client.py new file mode 100644 index 0000000..689624b --- /dev/null +++ b/openlane_scraper/openlane/client.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import base64 +import json +import logging +import random +import time +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlencode + +from playwright.sync_api import Page + +from ..core.config import OpenLaneConfig + +logger = logging.getLogger("openlane_scraper.openlane.client") + + +class OpenLaneRequestError(RuntimeError): + def __init__(self, message: str, *, status_code: int | None = None) -> None: + super().__init__(message) + self.status_code = status_code + + +@dataclass(slots=True) +class OpenLanePageResult: + page: int + records: list[dict[str, Any]] + raw_payload: dict[str, Any] + status_code: int + + +def _decode_access_token(token: str) -> dict[str, Any]: + # Декодирование JWT payload (без проверки подписи). + parts = token.split(".") + if len(parts) < 2: + return {} + payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4) + try: + return json.loads(base64.urlsafe_b64decode(payload_b64)) + except Exception: + return {} + + +class OpenLaneClient: + # HTTP-клиент OpenLane search API через Playwright. + + def __init__(self, authenticated_page: Page, config: OpenLaneConfig) -> None: + self.page = authenticated_page + self.config = config + self._user_id: str | None = None + self._dealership_id: str | None = None + self._init_ids_from_cookies() + + def _init_ids_from_cookies(self) -> None: + # Извлекаем user_id и dealership_id из access_token. + cookies = self.page.context.cookies("https://app.openlane.com") + for cookie in cookies: + if cookie.get("name") == "access_token": + payload = _decode_access_token(cookie["value"]) + self._user_id = str(payload.get("user_id", "")) + on_behalf = payload.get("on_behalf_of", {}) + self._dealership_id = str(on_behalf.get("dealership_id", "")) + if self._user_id and self._dealership_id: + logger.info( + "OpenLane client: user_id=%s dealership_id=%s", + self._user_id, self._dealership_id, + ) + return + logger.warning("OpenLane client: access_token cookie not found, API requests may fail") + + def _build_headers(self) -> dict[str, str]: + # Заголовки как у фронтенда OpenLane. + headers: dict[str, str] = { + "Accept": "application/json, application/vnd.backlotcars.v3", + "application": "webapp", + } + if self._user_id: + headers["x-rbz-user-id"] = self._user_id + if self._dealership_id: + headers["x-rbz-dealership-id"] = self._dealership_id + return headers + + def fetch_page(self, page: int) -> OpenLanePageResult: + params = { + "page": page, + "source_tab": self.config.source_tab, + "sale_types": self.config.sale_types, + } + if self.config.page_size > 0: + params["per_page"] = self.config.page_size + + self._sleep_with_jitter() + url = f"{self.config.api_search_url}?{urlencode(params)}" + logger.debug("OpenLane fetch page=%s url=%s", page, url) + + # fetch() с авторизацией в браузере. + headers_js = json.dumps(self._build_headers()) + fetch_result = self.page.evaluate( + """async ([url, headersJson]) => { + const headers = JSON.parse(headersJson); + const resp = await fetch(url, { + method: 'GET', + credentials: 'include', + headers: headers + }); + const text = await resp.text(); + return { status: resp.status, body: text }; + }""", + [url, headers_js], + ) + return self._parse_response(page, fetch_result) + + def fetch_vehicle_images(self, vehicle_id: str | int) -> list[dict[str, Any]]: + """Загружает изображения автомобиля через `/api/vehicles/{id}/images`.""" + if vehicle_id is None: + return [] + + vehicle_id_str = str(vehicle_id).strip() + if not vehicle_id_str: + return [] + + self._sleep_with_jitter() + url = f"https://app.openlane.com/api/vehicles/{vehicle_id_str}/images" + logger.debug("OpenLane fetch images vehicle_id=%s", vehicle_id_str) + + headers_js = json.dumps(self._build_headers()) + fetch_result = self.page.evaluate( + """async ([url, headersJson]) => { + const headers = JSON.parse(headersJson); + const resp = await fetch(url, { + method: 'GET', + credentials: 'include', + headers: headers + }); + const text = await resp.text(); + return { status: resp.status, body: text }; + }""", + [url, headers_js], + ) + + status_code = int(fetch_result.get("status", 0)) + text = str(fetch_result.get("body", "")) + if status_code == 404: + return [] + if status_code == 403: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden", + status_code=status_code, + ) + if status_code == 429: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests", + status_code=status_code, + ) + if status_code >= 500: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}", + status_code=status_code, + ) + if status_code == 401: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated", + status_code=status_code, + ) + if status_code >= 400: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}", + status_code=status_code, + ) + + if not text.strip(): + return [] + + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON" + ) from exc + + return self._extract_vehicle_images(payload) + + def fetch_vehicle_images_batch(self, vehicle_ids: list[str | int]) -> dict[str, list[dict[str, Any]]]: + """Пакетно загружает изображения автомобилей через `/api/vehicles/{id}/images`. + + Возвращает словарь `vehicle_id -> список image-объектов`. + """ + normalized_ids: list[str] = [] + seen: set[str] = set() + for vehicle_id in vehicle_ids: + vehicle_id_str = str(vehicle_id).strip() + if not vehicle_id_str or vehicle_id_str in seen: + continue + seen.add(vehicle_id_str) + normalized_ids.append(vehicle_id_str) + + if not normalized_ids: + return {} + + self._sleep_with_jitter() + headers_js = json.dumps(self._build_headers()) + fetch_results = self.page.evaluate( + """async ([vehicleIds, headersJson]) => { + const headers = JSON.parse(headersJson); + const tasks = vehicleIds.map(async (vehicleId) => { + try { + const url = `https://app.openlane.com/api/vehicles/${vehicleId}/images`; + const resp = await fetch(url, { + method: 'GET', + credentials: 'include', + headers: headers, + }); + const text = await resp.text(); + return { vehicleId, status: resp.status, body: text }; + } catch (error) { + return { vehicleId, status: 0, body: '', error: String(error) }; + } + }); + return await Promise.all(tasks); + }""", + [normalized_ids, headers_js], + ) + + result_map: dict[str, list[dict[str, Any]]] = {} + for item in fetch_results: + vehicle_id_str = str(item.get("vehicleId", "")).strip() + status_code = int(item.get("status", 0)) + text = str(item.get("body", "")) + + if not vehicle_id_str: + continue + + if status_code == 404: + result_map[vehicle_id_str] = [] + continue + if status_code == 403: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden", + status_code=status_code, + ) + if status_code == 429: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests", + status_code=status_code, + ) + if status_code >= 500 or status_code == 0: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}", + status_code=status_code if status_code else 500, + ) + if status_code == 401: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated", + status_code=status_code, + ) + if status_code >= 400: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}", + status_code=status_code, + ) + + if not text.strip(): + result_map[vehicle_id_str] = [] + continue + + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON" + ) from exc + + result_map[vehicle_id_str] = self._extract_vehicle_images(payload) + + return result_map + + def _parse_response(self, page: int, fetch_result: dict[str, Any]) -> OpenLanePageResult: + # Парсинг ответа fetch(): {status, body}. + status_code = int(fetch_result.get("status", 0)) + text = str(fetch_result.get("body", "")) + logger.debug( + "OpenLane response: page=%s status=%s body_len=%d body_preview=%.500s", + page, status_code, len(text), text[:500], + ) + if status_code == 403: + raise OpenLaneRequestError( + f"OpenLane page={page} returned 403 Forbidden — possible block, stopping", + status_code=status_code, + ) + if status_code == 429: + raise OpenLaneRequestError( + f"OpenLane page={page} returned 429 Too Many Requests — rate limited", + status_code=status_code, + ) + if status_code >= 500: + raise OpenLaneRequestError( + f"OpenLane page={page} returned server error {status_code}", + status_code=status_code, + ) + if status_code == 401: + raise OpenLaneRequestError( + f"OpenLane page={page} returned 401; session is not authenticated", + status_code=status_code, + ) + if status_code >= 400: + raise OpenLaneRequestError( + f"OpenLane page={page} returned unexpected status {status_code}", + status_code=status_code, + ) + + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + raise OpenLaneRequestError(f"OpenLane page={page} returned invalid JSON") from exc + + records = self._extract_records(payload) + logger.debug( + "OpenLane extracted: page=%s records=%d top_keys=%s", + page, len(records), list(payload.keys())[:10], + ) + return OpenLanePageResult( + page=page, + records=records, + raw_payload=payload, + status_code=status_code, + ) + + @staticmethod + def _extract_records(payload: dict[str, Any]) -> list[dict[str, Any]]: + candidates = [ + payload.get("results"), + payload.get("items"), + payload.get("data"), + payload.get("vehicles"), + ] + for candidate in candidates: + if isinstance(candidate, list): + return [item for item in candidate if isinstance(item, dict)] + if isinstance(candidate, dict): + nested_candidates = [ + candidate.get("results"), + candidate.get("items"), + candidate.get("vehicles"), + ] + for nested in nested_candidates: + if isinstance(nested, list): + return [item for item in nested if isinstance(item, dict)] + return [] + + @staticmethod + def _extract_vehicle_images(payload: Any) -> list[dict[str, Any]]: + candidates: list[Any] = [] + if isinstance(payload, dict): + candidates.extend( + [ + payload.get("vehicle_images"), + payload.get("images"), + payload.get("data"), + ] + ) + data = payload.get("data") + if isinstance(data, dict): + candidates.extend([data.get("vehicle_images"), data.get("images")]) + elif isinstance(payload, list): + candidates.append(payload) + + for candidate in candidates: + if isinstance(candidate, list): + return [item for item in candidate if isinstance(item, dict)] + return [] + + def _sleep_with_jitter(self) -> None: + low = min(self.config.throttle_min_seconds, self.config.throttle_max_seconds) + high = max(self.config.throttle_min_seconds, self.config.throttle_max_seconds) + time.sleep(random.uniform(low, high)) diff --git a/openlane_scraper/openlane/mapper.py b/openlane_scraper/openlane/mapper.py new file mode 100644 index 0000000..2c250ae --- /dev/null +++ b/openlane_scraper/openlane/mapper.py @@ -0,0 +1,424 @@ +"""Маппер: JSON-запись OpenLane API → CarRecord для сохранения в БД.""" + +from __future__ import annotations + +import hashlib +import logging +import re +from datetime import datetime, timezone +from typing import Any + +from ..storage.schemas import CarRecord, ImageRecord + +logger = logging.getLogger("openlane_scraper.openlane.mapper") + +# Маппинг body_type из OpenLane в наш enum. +BODY_MAP: dict[str, str] = { + "sedan": "SEDAN", + "coupe": "COUPE", + "suv": "SUV", + "sport utility": "SUV", + "crossover": "SUV", + "hatchback": "HATCHBACK", + "minivan": "MINIVAN", + "van": "MINIVAN", + "wagon": "STATION_WAGON", + "station wagon": "STATION_WAGON", + "pickup": "PICKUP", + "truck": "TRUCK", + "convertible": "OPEN", + "cabriolet": "OPEN", + "roadster": "OPEN", + "rv": "RV", + "motorhome": "RV", +} + +DRIVE_MAP: dict[str, str] = { + "fwd": "FWD", + "front wheel drive": "FWD", + "front-wheel drive": "FWD", + "rwd": "RWD", + "rear wheel drive": "RWD", + "rear-wheel drive": "RWD", + "awd": "4WD", + "4wd": "4WD", + "all wheel drive": "4WD", + "all-wheel drive": "4WD", + "4x4": "4WD", + "2wd": "2WD", + "two wheel drive": "2WD", +} + +GEARBOX_MAP: dict[str, str] = { + "automatic": "AT", + "auto": "AT", + "at": "AT", + "manual": "MT", + "mt": "MT", + "cvt": "CVT", + "continuously variable": "CVT", + "electric": "EV", + "ev": "EV", +} + +_MILEAGE_RE = re.compile(r"[\d,]+") +_ENGINE_RE = re.compile(r"(\d+\.?\d*)\s*[lL]") + + +def _safe_str(value: Any, default: str = "") -> str: + if value is None: + return default + return str(value).strip() or default + + +def _safe_int(value: Any) -> int | None: + if value is None: + return None + try: + cleaned = str(value).replace(",", "").strip() + if not cleaned: + return None + return int(float(cleaned)) + except (ValueError, TypeError): + return None + + +def _normalize_enum(raw: Any, mapping: dict[str, str], default: str = "NA") -> str: + if not raw: + return default + key = str(raw).strip().lower() + return mapping.get(key, default) + + +def _generate_parser_id(origin_id: str) -> str: + return hashlib.sha256(origin_id.encode()).hexdigest()[:40] + + +def _generate_slug(year: int | None, brand: str, model: str, origin_id: str) -> str: + parts = [] + if year: + parts.append(str(year)) + parts.append(brand.lower()) + parts.append(model.lower()) + parts.append(origin_id.replace(":", "-")) + slug = "-".join(parts) + slug = re.sub(r"[^a-z0-9\-]", "-", slug) + slug = re.sub(r"-+", "-", slug).strip("-") + return slug[:200] + + +def _build_openlane_origin_id(raw_id: Any) -> str | None: + """Строит origin_id в формате openlane:id.""" + normalized = _safe_str(raw_id) + if not normalized: + return None + if normalized.lower().startswith("openlane:"): + normalized = _safe_str(normalized.split(":", 1)[1]) + if not normalized: + return None + return f"openlane:{normalized}" + + +def _extract_nested(record: dict[str, Any], *keys: str) -> Any: + """Извлекает значение из вложенного словаря по цепочке ключей.""" + obj: Any = record + for key in keys: + if isinstance(obj, dict): + obj = obj.get(key) + else: + return None + return obj + + +def _extract_images(record: dict[str, Any]) -> list[ImageRecord]: + """Извлекает изображения из записи OpenLane.""" + images: list[ImageRecord] = [] + seen_urls: set[str] = set() + + raw_images = ( + record.get("images") + or record.get("photos") + or record.get("media", {}).get("images") + or record.get("image_urls") + or [] + ) + + if isinstance(raw_images, list): + for idx, img in enumerate(raw_images): + if isinstance(img, str): + url = img.strip() + if url and url not in seen_urls: + seen_urls.add(url) + images.append(ImageRecord( + fullres_image=url, + preview_image=url, + order_index=idx, + )) + elif isinstance(img, dict): + fullres = _safe_str( + img.get("full") or img.get("fullres") or img.get("url") + or img.get("original") or img.get("large") or img.get("href") + or img.get("large_resolution_url") + ) + preview = _safe_str( + img.get("thumbnail") or img.get("thumb") or img.get("preview") + or img.get("small") or img.get("low_resolution_url") + or fullres + ) + if fullres and fullres not in seen_urls: + seen_urls.add(fullres) + images.append(ImageRecord( + fullres_image=fullres, + preview_image=preview, + order_index=idx, + )) + + # Fallback: одиночное изображение. + if not images: + single = ( + record.get("image_url") + or record.get("image") + or record.get("large_resolution_url") + or record.get("low_resolution_url") + or record.get("thumbnail") + or record.get("photo_url") + or _extract_nested(record, "media", "primary") + ) + if single and isinstance(single, str) and single.strip(): + images.append(ImageRecord( + fullres_image=single.strip(), + preview_image=single.strip(), + order_index=0, + )) + + return images + + +def _parse_mileage(raw: Any) -> int: + if raw is None: + return 0 + if isinstance(raw, (int, float)): + return max(0, int(raw)) + text = str(raw) + match = _MILEAGE_RE.search(text) + if match: + try: + return max(0, int(match.group().replace(",", ""))) + except ValueError: + pass + return 0 + + +def _parse_engine_volume_cc(raw: Any) -> int | None: + """Парсит объём двигателя и возвращает значение в кубических сантиметрах.""" + if raw is None: + return None + if isinstance(raw, (int, float)): + value = float(raw) + # Если значение < 20 — скорее всего это литры, конвертируем в cc. + if 0 < value < 20: + return int(value * 1000) + if value >= 100: + return int(value) + return None + text = str(raw) + match = _ENGINE_RE.search(text) + if match: + liters = float(match.group(1)) + return int(liters * 1000) + return None + + +def map_openlane_record(record: dict[str, Any]) -> CarRecord | None: + """Маппит одну запись из OpenLane API в CarRecord. + + Возвращает None, если запись не содержит минимально необходимых данных. + """ + # Извлекаем идентификатор. + raw_id = ( + record.get("id") + or record.get("vehicle_id") + or record.get("listing_id") + or record.get("vin") + ) + if not raw_id: + logger.debug("Skipping record without id: %s", record.get("vin", "unknown")) + return None + + origin_id = _build_openlane_origin_id(raw_id) + if not origin_id: + logger.debug("Skipping record with malformed id: %s", raw_id) + return None + + # Бренд и модель — обязательные поля. + brand = _safe_str( + record.get("make") + or record.get("brand") + or record.get("manufacturer") + or _extract_nested(record, "vehicle", "make") + ).upper() + + model = _safe_str( + record.get("model") + or record.get("model_name") + or _extract_nested(record, "vehicle", "model") + ).upper() + + if not brand or not model: + logger.debug("Skipping record without brand/model: %s", origin_id) + return None + + year = _safe_int( + record.get("year") + or record.get("model_year") + or _extract_nested(record, "vehicle", "year") + ) + + price = _safe_int( + record.get("price") + or record.get("current_bid") + or record.get("buy_now_price") + or record.get("asking_price") + or record.get("sale_price") + or _extract_nested(record, "pricing", "current") + or _extract_nested(record, "pricing", "buy_now") + ) + + currency = _safe_str( + record.get("currency") + or record.get("currency_code") + or _extract_nested(record, "pricing", "currency"), + "USD", + ).upper() + if currency not in {"JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD"}: + currency = "USD" + + mileage = _parse_mileage( + record.get("mileage") + or record.get("odometer") + or record.get("odometer_reading") + or _extract_nested(record, "vehicle", "mileage") + ) + + color = _safe_str( + record.get("color") + or record.get("exterior_color") + or _extract_nested(record, "vehicle", "color"), + "other", + ).lower() + + body_type = _normalize_enum( + record.get("body_type") + or record.get("body_style") + or record.get("vehicle_type") + or _extract_nested(record, "vehicle", "body_type"), + BODY_MAP, + "OTHER", + ) + + drive = _normalize_enum( + record.get("drive_type") + or record.get("drivetrain") + or record.get("drive") + or _extract_nested(record, "vehicle", "drivetrain"), + DRIVE_MAP, + ) + + gearbox = _normalize_enum( + record.get("transmission") + or record.get("gearbox") + or _extract_nested(record, "vehicle", "transmission"), + GEARBOX_MAP, + ) + + engine_volume = _parse_engine_volume_cc( + record.get("engine") + or record.get("engine_size") + or record.get("displacement") + or _extract_nested(record, "vehicle", "engine") + ) + + # URL записи на OpenLane. + origin_url = _safe_str( + record.get("url") + or record.get("listing_url") + or record.get("detail_url") + or record.get("permalink") + ) + if not origin_url: + origin_url = f"https://app.openlane.com/vehicles/{raw_id}" + + country = _safe_str( + record.get("country") + or record.get("location_country") + or _extract_nested(record, "location", "country"), + "US", + ).upper() + if country not in {"JP", "KR", "US", "CA", "NA"}: + country = "US" + + is_damaged = bool( + record.get("is_damaged") + or record.get("has_damage") + or record.get("damage_type") + ) + + vin = _safe_str(record.get("vin") or _extract_nested(record, "vehicle", "vin")) + evaluation = vin if vin else None + + selling_type = "AUCTION" + sale_type = _safe_str(record.get("sale_type") or record.get("listing_type")).lower() + if sale_type in {"buy_now", "fixed_price", "stock"}: + selling_type = "STOCK" + elif sale_type in {"tender"}: + selling_type = "TENDER" + + images = _extract_images(record) + + parser_id = _generate_parser_id(origin_id) + slug = _generate_slug(year, brand, model, origin_id) + + return CarRecord( + parser_id=parser_id, + brand=brand, + model=model, + year=year, + price=price, + currency=currency, + mileage=mileage, + country=country, + is_sold=False, + color=color, + drive=drive if drive != "NA" else None, + gearbox=gearbox if gearbox != "NA" else None, + body_type=body_type, + engine_volume=engine_volume, + selling_type=selling_type, + origin="OPENLANE", + origin_url=origin_url, + origin_id=origin_id, + is_damaged=is_damaged, + evaluation=evaluation, + slug=slug, + images=images, + ) + + +def map_openlane_records(records: list[dict[str, Any]]) -> list[CarRecord]: + """Маппит список записей OpenLane API в список CarRecord. + + Пропускает записи без минимально необходимых данных. + """ + result: list[CarRecord] = [] + for record in records: + try: + car = map_openlane_record(record) + if car is not None: + result.append(car) + except Exception: + logger.warning( + "Failed to map OpenLane record id=%s", + record.get("id", "unknown"), + exc_info=True, + ) + return result diff --git a/openlane_scraper/openlane/runner.py b/openlane_scraper/openlane/runner.py new file mode 100644 index 0000000..8bbc021 --- /dev/null +++ b/openlane_scraper/openlane/runner.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +import logging +import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from playwright.sync_api import sync_playwright + +from ..browser.factory import BrowserFactory +from ..core.config import Settings +from ..core.logs import set_trace_id, setup_logging +from .auth import OpenLaneAuthenticator +from .checkpoint import OpenLaneCheckpoint, OpenLaneCheckpointStore +from .client import OpenLaneClient, OpenLanePageResult, OpenLaneRequestError +from .writer import OpenLaneResultWriter + +logger = logging.getLogger("openlane_scraper.openlane.runner") + + +@dataclass(slots=True) +class OpenLaneScrapeResult: + jsonl_path: str + aggregated_path: str + checkpoint_path: str + storage_state_path: str + completed_pages: list[int] + failed_pages: list[int] + total_records: int + elapsed_seconds: float + + +class OpenLaneScrapeRunner: + def __init__(self, runtime_settings: Settings | None = None) -> None: + self.settings = runtime_settings or Settings() + setup_logging(self.settings.log_level, self.settings.log_file) + self.trace_id = f"openlane-{uuid.uuid4().hex[:8]}" + set_trace_id(self.trace_id) + self.openlane = self.settings.openlane + self.browser_factory = BrowserFactory(self.settings) + self.authenticator = OpenLaneAuthenticator(self.openlane) + self.writer = OpenLaneResultWriter( + self.openlane.jsonl_output, + self.openlane.aggregated_output, + ) + self.checkpoint_store = OpenLaneCheckpointStore(self.openlane.checkpoint_file) + + def run( + self, + *, + max_pages: int | None = None, + concurrency: int | None = None, + resume: bool = False, + checkpoint_every_pages: int | None = None, + ) -> OpenLaneScrapeResult: + started_at = time.perf_counter() + max_pages = max_pages or self.openlane.max_pages + concurrency = max(1, min(concurrency or self.openlane.concurrency, 5)) + checkpoint_every_pages = checkpoint_every_pages or self.openlane.checkpoint_every_pages + + checkpoint = self.checkpoint_store.load(max_pages=max_pages) if resume else OpenLaneCheckpoint(max_pages=max_pages) + checkpoint.max_pages = max_pages + + if not resume: + self._reset_outputs() + + logger.info( + "Starting OpenLane scrape: max_pages=%s concurrency=%s resume=%s checkpoint_every_pages=%s", + max_pages, + concurrency, + resume, + checkpoint_every_pages, + ) + + with sync_playwright() as playwright: + browser = self.browser_factory.create_browser(playwright) + try: + auth_pages = [] + for worker_index in range(concurrency): + storage_state_path = self.openlane.storage_state_file if Path(self.openlane.storage_state_file).exists() else None + context = self.browser_factory.create_context(browser, storage_state_path=storage_state_path) + auth_page = self.authenticator.bootstrap_authenticated_context(context) + auth_pages.append(auth_page) + logger.info("OpenLane worker session initialized: worker=%s", worker_index + 1) + + pending_pages = [ + page for page in range(1, max_pages + 1) + if page not in checkpoint.completed_set + ] + self._run_workers(auth_pages, pending_pages, checkpoint, checkpoint_every_pages) + finally: + browser.close() + + self.checkpoint_store.save(checkpoint) + aggregated = self.writer.finalize( + max_pages=max_pages, + completed_pages=checkpoint.completed_pages, + failed_pages=checkpoint.failed_pages, + ) + elapsed_seconds = time.perf_counter() - started_at + logger.info( + "OpenLane scrape finished: completed_pages=%s failed_pages=%s total_records=%s elapsed=%.2fs", + len(set(checkpoint.completed_pages)), + len(set(checkpoint.failed_pages)), + aggregated["total_records"], + elapsed_seconds, + ) + return OpenLaneScrapeResult( + jsonl_path=str(Path(self.openlane.jsonl_output)), + aggregated_path=str(Path(self.openlane.aggregated_output)), + checkpoint_path=str(Path(self.openlane.checkpoint_file)), + storage_state_path=str(Path(self.openlane.storage_state_file)), + completed_pages=sorted(set(checkpoint.completed_pages)), + failed_pages=sorted(set(checkpoint.failed_pages)), + total_records=int(aggregated["total_records"]), + elapsed_seconds=elapsed_seconds, + ) + + def _run_workers( + self, + auth_pages: list, + pending_pages: list[int], + checkpoint: OpenLaneCheckpoint, + checkpoint_every_pages: int, + ) -> None: + # Последовательная обработка страниц (sync API — однопоточный). + started_at = time.perf_counter() + + for idx, page_num in enumerate(pending_pages): + worker_index = (idx % len(auth_pages)) + 1 + auth_page = auth_pages[idx % len(auth_pages)] + + try: + result = self._fetch_page_with_retry(auth_page, page_num, worker_index) + self._handle_success(result, checkpoint, started_at) + except Exception as exc: + self._handle_failure(page_num, exc, checkpoint, started_at) + + processed_count = len(set(checkpoint.completed_pages)) + len(set(checkpoint.failed_pages)) + if processed_count and processed_count % checkpoint_every_pages == 0: + checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat() + self.checkpoint_store.save(checkpoint) + logger.info( + "OpenLane checkpoint saved: processed=%s completed=%s failed=%s", + processed_count, + len(set(checkpoint.completed_pages)), + len(set(checkpoint.failed_pages)), + ) + + def _fetch_page_with_retry(self, authenticated_page, page: int, worker_index: int) -> OpenLanePageResult: + set_trace_id(f"{self.trace_id}-w{worker_index}-p{page}") + client = OpenLaneClient(authenticated_page, self.openlane) + retry_schedule = self.openlane.retry_schedule_seconds + + for attempt in range(len(retry_schedule) + 1): + try: + logger.debug("OpenLane fetch attempt: page=%s worker=%s attempt=%s", page, worker_index, attempt + 1) + return client.fetch_page(page) + except OpenLaneRequestError as exc: + is_retryable = exc.status_code in {403, 429} or (exc.status_code is not None and exc.status_code >= 500) + if not is_retryable or attempt >= len(retry_schedule): + logger.error( + "OpenLane fetch failed permanently: page=%s worker=%s status=%s error=%s", + page, + worker_index, + exc.status_code, + exc, + ) + raise + delay = retry_schedule[attempt] + logger.warning( + "OpenLane retry scheduled: page=%s worker=%s status=%s delay=%.1fs attempt=%s", + page, + worker_index, + exc.status_code, + delay, + attempt + 1, + ) + time.sleep(delay) + except Exception: + if attempt >= len(retry_schedule): + raise + delay = retry_schedule[attempt] + logger.warning( + "OpenLane transient error retry: page=%s worker=%s delay=%.1fs attempt=%s", + page, + worker_index, + delay, + attempt + 1, + exc_info=True, + ) + time.sleep(delay) + + raise RuntimeError(f"OpenLane page {page} exhausted retries") + + def _handle_success( + self, + result: OpenLanePageResult, + checkpoint: OpenLaneCheckpoint, + started_at: float, + ) -> None: + written = self.writer.append_page(result.page, result.records) + checkpoint.completed_pages = sorted(set(checkpoint.completed_pages) | {result.page}) + checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) - {result.page}) + checkpoint.total_records += written + checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat() + self._log_progress(result.page, checkpoint, started_at, written, None) + + def _handle_failure( + self, + page: int, + exc: Exception, + checkpoint: OpenLaneCheckpoint, + started_at: float, + ) -> None: + checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) | {page}) + checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat() + self._log_progress(page, checkpoint, started_at, 0, exc) + + def _log_progress( + self, + page: int, + checkpoint: OpenLaneCheckpoint, + started_at: float, + records_written: int, + exc: Exception | None, + ) -> None: + completed = len(set(checkpoint.completed_pages)) + failed = len(set(checkpoint.failed_pages)) + elapsed_seconds = max(time.perf_counter() - started_at, 0.001) + pages_per_minute = (completed + failed) / elapsed_seconds * 60.0 + if exc is None: + logger.info( + "OpenLane progress: page=%s completed=%s failed=%s records=%s total_records=%s speed=%.2f pages/min", + page, + completed, + failed, + records_written, + checkpoint.total_records, + pages_per_minute, + ) + else: + logger.error( + "OpenLane page error: page=%s completed=%s failed=%s total_records=%s speed=%.2f pages/min error=%s", + page, + completed, + failed, + checkpoint.total_records, + pages_per_minute, + exc, + ) + + def interactive_login(self) -> str: + setup_logging(self.settings.log_level, self.settings.log_file) + with sync_playwright() as playwright: + browser = self.browser_factory.create_browser(playwright) + try: + context = self.browser_factory.create_context(browser) + return self.authenticator.interactive_login_and_persist(context) + finally: + browser.close() + + def _reset_outputs(self) -> None: + for raw_path in ( + self.openlane.jsonl_output, + self.openlane.aggregated_output, + self.openlane.checkpoint_file, + ): + path = Path(raw_path) + if path.exists(): + path.unlink() diff --git a/openlane_scraper/openlane/writer.py b/openlane_scraper/openlane/writer.py new file mode 100644 index 0000000..d6d79f8 --- /dev/null +++ b/openlane_scraper/openlane/writer.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +class OpenLaneResultWriter: + def __init__(self, jsonl_path: str | Path, aggregated_path: str | Path) -> None: + self.jsonl_path = Path(jsonl_path) + self.aggregated_path = Path(aggregated_path) + + def ensure_parent_dirs(self) -> None: + self.jsonl_path.parent.mkdir(parents=True, exist_ok=True) + self.aggregated_path.parent.mkdir(parents=True, exist_ok=True) + + def append_page(self, page: int, records: list[dict[str, Any]]) -> int: + self.ensure_parent_dirs() + written = 0 + with self.jsonl_path.open("a", encoding="utf-8") as fh: + for record in records: + payload = { + "page": page, + "record": record, + } + fh.write(json.dumps(payload, ensure_ascii=False) + "\n") + written += 1 + return written + + def finalize(self, *, max_pages: int, completed_pages: list[int], failed_pages: list[int]) -> dict[str, Any]: + self.ensure_parent_dirs() + records: list[dict[str, Any]] = [] + if self.jsonl_path.exists(): + with self.jsonl_path.open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + item = json.loads(line) + records.append(item) + + summary = { + "max_pages": max_pages, + "completed_pages": sorted(completed_pages), + "failed_pages": sorted(failed_pages), + "total_pages_completed": len(set(completed_pages)), + "total_pages_failed": len(set(failed_pages)), + "total_records": len(records), + "items": records, + } + self.aggregated_path.write_text( + json.dumps(summary, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + return summary diff --git a/openlane_scraper/scraper.py b/openlane_scraper/scraper.py new file mode 100644 index 0000000..e14d3e3 --- /dev/null +++ b/openlane_scraper/scraper.py @@ -0,0 +1,514 @@ +"""Главный оркестратор скрапинга OpenLane с сохранением в БД. + +Использует OpenLane API для получения данных и PersistenceService для записи в PostgreSQL. +Batched upsert, SyncRun-аудит, early-stop при отсутствии новых записей. +""" + +from __future__ import annotations + +import logging +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from playwright.sync_api import sync_playwright + +from .browser.factory import BrowserFactory +from .core.config import Settings +from .core.logs import set_trace_id, setup_logging +from .core.runtime_config import RuntimeConfig, apply_filters, load_runtime_config +from .openlane.auth import OpenLaneAuthenticator +from .openlane.client import OpenLaneClient, OpenLanePageResult, OpenLaneRequestError +from .openlane.mapper import map_openlane_records +from .storage.db import PersistenceService +from .storage.schemas import CarRecord, ImageRecord + +logger = logging.getLogger("openlane_scraper.scraper") + + +class OpenLaneScraper: + """Оркестратор: OpenLane API → маппинг → DB upsert.""" + + def __init__(self, runtime_settings: Settings | None = None) -> None: + self.settings = runtime_settings or Settings() + setup_logging(self.settings.log_level, self.settings.log_file) + self.trace_id = f"openlane-sync-{uuid.uuid4().hex[:8]}" + set_trace_id(self.trace_id) + self.openlane_config = self.settings.openlane + self.browser_factory = BrowserFactory(self.settings) + self.authenticator = OpenLaneAuthenticator(self.openlane_config) + self.persistence = PersistenceService(self.settings) + self.runtime_config = load_runtime_config(self.settings.runtime_config_file) + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + @staticmethod + def _normalize_origin_id(raw_id: Any) -> str | None: + if raw_id is None: + return None + normalized = str(raw_id).strip() + if not normalized: + return None + if normalized.lower().startswith("openlane:"): + normalized = normalized.split(":", 1)[1].strip() + if not normalized: + return None + return f"openlane:{normalized}" + + @staticmethod + def _build_image_records(images_payload: list[dict[str, Any]]) -> list[ImageRecord]: + images: list[ImageRecord] = [] + seen: set[str] = set() + for idx, img in enumerate(images_payload): + fullres = str( + img.get("url") + or img.get("large_resolution_url") + or img.get("image_large") + or img.get("image") + or img.get("full") + or img.get("fullres") + or "" + ).strip() + if not fullres or fullres in seen: + continue + preview = str( + img.get("low_resolution_url") + or img.get("medium_resolution_url") + or img.get("thumbnail_url") + or img.get("thumbnail") + or img.get("thumb") + or img.get("image") + or fullres + ).strip() + seen.add(fullres) + images.append( + ImageRecord( + fullres_image=fullres, + preview_image=preview or fullres, + order_index=idx, + ) + ) + return images + + @staticmethod + def _merge_image_records(existing: list[ImageRecord], fetched: list[ImageRecord]) -> list[ImageRecord]: + merged: list[ImageRecord] = [] + seen: set[str] = set() + + for source in (fetched, existing): + for image in source: + key = image.fullres_image.strip() + if not key or key in seen: + continue + seen.add(key) + merged.append(image) + + for idx, image in enumerate(merged): + image.order_index = idx + + return merged + + def _fetch_vehicle_images_with_retry( + self, + client: OpenLaneClient, + vehicle_id: str, + ) -> list[dict[str, Any]]: + retry_schedule = self.openlane_config.retry_schedule_seconds + for attempt in range(len(retry_schedule) + 1): + try: + return client.fetch_vehicle_images(vehicle_id) + except OpenLaneRequestError as exc: + if exc.status_code in {401, 403}: + raise + is_retryable = exc.status_code == 429 or ( + exc.status_code is not None and exc.status_code >= 500 + ) + if not is_retryable or attempt >= len(retry_schedule): + raise + delay = retry_schedule[attempt] + if exc.status_code == 429: + delay = max(delay * 2, 8.0) + logger.warning( + "Retry vehicle images vehicle_id=%s status=%s delay=%.1fs attempt=%d/%d", + vehicle_id, + exc.status_code, + delay, + attempt + 1, + len(retry_schedule), + ) + time.sleep(delay) + + return [] + + def _enrich_car_records_with_images( + self, + authenticated_page, + raw_records: list[dict[str, Any]], + car_records: list[CarRecord], + ) -> int: + if not car_records: + return 0 + + raw_by_origin: dict[str, dict[str, Any]] = {} + for raw in raw_records: + raw_id = ( + raw.get("id") + or raw.get("vehicle_id") + or raw.get("listing_id") + or raw.get("vin") + ) + origin_id = self._normalize_origin_id(raw_id) + if origin_id: + raw_by_origin[origin_id] = raw + + client = OpenLaneClient(authenticated_page, self.openlane_config) + images_added = 0 + + cars_by_vehicle_id: dict[str, CarRecord] = {} + for car in car_records: + raw = raw_by_origin.get(car.origin_id, {}) + vehicle_id = raw.get("id") or raw.get("vehicle_id") or raw.get("listing_id") + if not vehicle_id: + # fallback: пробуем id из origin_id. + vehicle_id = car.origin_id.split(":", 1)[1] if ":" in car.origin_id else None + if not vehicle_id: + continue + cars_by_vehicle_id[str(vehicle_id)] = car + + if not cars_by_vehicle_id: + return 0 + + vehicle_ids = list(cars_by_vehicle_id.keys()) + try: + images_map = self._fetch_vehicle_images_batch_with_retry(client, vehicle_ids) + except OpenLaneRequestError as exc: + if exc.status_code in {401, 403}: + raise + logger.warning("Vehicle images batch fetch failed: %s", exc) + return 0 + except Exception as exc: + logger.warning("Vehicle images batch fetch failed: %s", exc) + return 0 + + for vehicle_id, car in cars_by_vehicle_id.items(): + images_payload = images_map.get(vehicle_id, []) + fetched_records = self._build_image_records(images_payload) + if not fetched_records: + continue + + before_count = len(car.images) + # Source of truth: /api/vehicles/{id}/images. + car.images = fetched_records + after_count = len(car.images) + if after_count > before_count: + images_added += after_count - before_count + + return images_added + + def _fetch_vehicle_images_batch_with_retry( + self, + client: OpenLaneClient, + vehicle_ids: list[str], + ) -> dict[str, list[dict[str, Any]]]: + retry_schedule = self.openlane_config.retry_schedule_seconds + for attempt in range(len(retry_schedule) + 1): + try: + return client.fetch_vehicle_images_batch(vehicle_ids) + except OpenLaneRequestError as exc: + if exc.status_code in {401, 403}: + raise + is_retryable = exc.status_code == 429 or ( + exc.status_code is not None and exc.status_code >= 500 + ) + if not is_retryable or attempt >= len(retry_schedule): + raise + delay = retry_schedule[attempt] + if exc.status_code == 429: + delay = max(delay * 2, 8.0) + logger.warning( + "Retry vehicle images batch size=%d status=%s delay=%.1fs attempt=%d/%d", + len(vehicle_ids), + exc.status_code, + delay, + attempt + 1, + len(retry_schedule), + ) + time.sleep(delay) + + return {} + + def init_db(self) -> dict[str, Any]: + self.persistence.create_tables() + return {"status": "ok", "message": "DB tables created"} + + def sync_listing( + self, + *, + lane: str | None = None, + limit: int | None = None, + only_new: bool | None = None, + max_pages: int | None = None, + concurrency: int | None = None, + ) -> dict[str, Any]: + """Полный цикл синхронизации: OpenLane API → фильтры → маппинг → DB. + + Параметры берутся из аргументов → runtime_config.json → config.py (в этом порядке). + """ + started_at = time.perf_counter() + rc = self.runtime_config + + # Параметры: аргумент → runtime_config → config default. + lane = lane or rc.sync.lane or "openlane_marketplace" + if limit is None: + limit = rc.sync.limit + if limit is None and self.openlane_config.max_cars_limit > 0: + limit = self.openlane_config.max_cars_limit + if only_new is None: + only_new = rc.sync.only_new + effective_only_new = only_new if only_new is not None else False + max_pages = max_pages or self.openlane_config.max_pages + concurrency = max(1, min(concurrency or self.openlane_config.concurrency, 5)) + + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane) + + total_upserted = 0 + total_failed = 0 + total_images = 0 + total_discovered = 0 + total_skipped = 0 + all_origin_ids: set[str] = set() + completed_pages: list[int] = [] + failed_pages: list[int] = [] + status = "success" + error_summary: str | None = None + + # Предзагружаем известные origin_id для раннего пропуска. + known_ids: set[str] = set() + if effective_only_new: + known_ids = self.persistence.get_all_origin_ids_for_lane(prefix="openlane:") + + logger.info( + "sync_listing started: lane=%s max_pages=%s concurrency=%s only_new=%s known=%d", + lane, max_pages, concurrency, effective_only_new, len(known_ids), + ) + + try: + with sync_playwright() as playwright: + browser = self.browser_factory.create_browser(playwright) + try: + contexts = [] + auth_pages = [] + for i in range(concurrency): + storage_state_path = ( + self.openlane_config.storage_state_file + if Path(self.openlane_config.storage_state_file).exists() + else None + ) + ctx = self.browser_factory.create_context(browser, storage_state_path=storage_state_path) + auth_page = self.authenticator.bootstrap_authenticated_context(ctx) + contexts.append(ctx) + auth_pages.append(auth_page) + logger.info("Worker session initialized: worker=%d", i + 1) + + # Последовательная обработка страниц (по одной на контекст). + # Для каждой страницы: fetch → map → upsert. + context_idx = 0 + consecutive_all_known = 0 + consecutive_failures = 0 + early_stop_threshold = 3 # Остановка если 3 страницы подряд без новых записей. + failure_circuit_breaker = 2 # Остановка если 2 подряд ошибки API. + + for page_num in range(1, max_pages + 1): + if limit is not None and total_upserted >= limit: + logger.info("Reached limit=%d, stopping", limit) + break + + if consecutive_failures >= failure_circuit_breaker: + logger.error( + "Circuit breaker: %d consecutive failures — stopping to protect account", + consecutive_failures, + ) + status = "aborted" + break + + ctx = auth_pages[context_idx % len(auth_pages)] + context_idx += 1 + + try: + page_result = self._fetch_page_with_retry(ctx, page_num) + consecutive_failures = 0 # Сброс при успехе. + except Exception as exc: + logger.error("Page %d fetch failed: %s", page_num, exc) + failed_pages.append(page_num) + total_failed += 1 + consecutive_failures += 1 + continue + + if not page_result.records: + logger.info("Page %d returned 0 records — end of listing", page_num) + completed_pages.append(page_num) + break + + # Применяем runtime-фильтры к сырым записям. + filtered_records = apply_filters(page_result.records, rc.filters) + if len(filtered_records) < len(page_result.records): + logger.debug( + "Page %d: %d/%d records passed runtime filters", + page_num, len(filtered_records), len(page_result.records), + ) + + # Маппинг JSON → CarRecord. + car_records = map_openlane_records(filtered_records) + total_discovered += len(page_result.records) + + if not car_records: + logger.warning("Page %d: all %d records failed mapping", page_num, len(page_result.records)) + completed_pages.append(page_num) + continue + + # Фильтрация уже известных записей. + if effective_only_new and known_ids: + new_records = [r for r in car_records if r.origin_id not in known_ids] + skipped = len(car_records) - len(new_records) + total_skipped += skipped + if skipped: + logger.debug("Page %d: skipped %d already known", page_num, skipped) + if not new_records: + consecutive_all_known += 1 + completed_pages.append(page_num) + if consecutive_all_known >= early_stop_threshold: + logger.info( + "Early stop: %d consecutive pages with all known records", + consecutive_all_known, + ) + break + continue + else: + consecutive_all_known = 0 + car_records = new_records + + # Применяем limit. + if limit is not None: + remaining = limit - total_upserted + car_records = car_records[:remaining] + + # Дозапрашиваем фото для машин, у которых их нет в search payload. + try: + added_images = self._enrich_car_records_with_images(ctx, filtered_records, car_records) + if added_images: + logger.debug("Page %d: enriched %d images via vehicle images API", page_num, added_images) + except OpenLaneRequestError as exc: + if exc.status_code in {401, 403}: + raise + logger.warning("Page %d image enrichment skipped: %s", page_num, exc) + + # Upsert батчем. + try: + upsert_result = self.persistence.upsert_cars_batch(car_records) + page_upserted = upsert_result.get("inserted", 0) + upsert_result.get("updated", 0) + page_images = upsert_result.get("images_upserted", 0) + total_upserted += page_upserted + total_images += page_images + for r in car_records: + all_origin_ids.add(r.origin_id) + known_ids.add(r.origin_id) + logger.info( + "Page %d: %d records → %d upserted, %d images", + page_num, len(car_records), page_upserted, page_images, + ) + except Exception as exc: + logger.error("Page %d upsert failed: %s", page_num, exc, exc_info=True) + total_failed += len(car_records) + failed_pages.append(page_num) + continue + + completed_pages.append(page_num) + + finally: + browser.close() + + except Exception as exc: + status = "failed" + error_summary = str(exc)[:500] + logger.error("sync_listing failed: %s", exc, exc_info=True) + + elapsed = time.perf_counter() - started_at + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=total_discovered, + cars_upserted=total_upserted, + cars_failed=total_failed, + images_upserted=total_images, + error_summary=error_summary, + ) + + result = { + "run_id": run_id, + "status": status, + "lane": lane, + "total_discovered": total_discovered, + "cars_upserted": total_upserted, + "cars_failed": total_failed, + "images_upserted": total_images, + "skipped_existing": total_skipped, + "completed_pages": len(completed_pages), + "failed_pages": len(failed_pages), + "elapsed_seconds": round(elapsed, 2), + "full_scan_completed": status == "success", + } + logger.info("sync_listing finished: %s", result) + return result + + def _fetch_page_with_retry(self, authenticated_page, page: int) -> OpenLanePageResult: + client = OpenLaneClient(authenticated_page, self.openlane_config) + retry_schedule = self.openlane_config.retry_schedule_seconds + + for attempt in range(len(retry_schedule) + 1): + try: + return client.fetch_page(page) + except OpenLaneRequestError as exc: + # 403 — возможная блокировка, стоп. + if exc.status_code == 403: + logger.error( + "STOP: page=%d returned 403 Forbidden — aborting to protect account", + page, + ) + raise + # 401 — сессия истекла. + if exc.status_code == 401: + raise + # 429 — rate limit, ретрай с увеличенным backoff. + is_retryable = exc.status_code == 429 or ( + exc.status_code is not None and exc.status_code >= 500 + ) + if not is_retryable or attempt >= len(retry_schedule): + raise + delay = retry_schedule[attempt] + if exc.status_code == 429: + delay = max(delay * 3, 15.0) # 429 → утроенная задержка + logger.warning( + "Retry page=%d status=%s delay=%.1fs attempt=%d/%d", + page, exc.status_code, delay, attempt + 1, len(retry_schedule), + ) + time.sleep(delay) + except Exception: + # Неожиданная ошибка — одна попытка. + if attempt >= min(1, len(retry_schedule)): + raise + delay = retry_schedule[attempt] if attempt < len(retry_schedule) else 5.0 + logger.warning( + "Transient error page=%d delay=%.1fs attempt=%d", + page, delay, attempt + 1, + exc_info=True, + ) + time.sleep(delay) + + raise RuntimeError(f"Page {page} exhausted retries") diff --git a/iaai_scraper/parsing/__init__.py b/openlane_scraper/storage/__init__.py similarity index 100% rename from iaai_scraper/parsing/__init__.py rename to openlane_scraper/storage/__init__.py diff --git a/iaai_scraper/storage/db.py b/openlane_scraper/storage/db.py similarity index 98% rename from iaai_scraper/storage/db.py rename to openlane_scraper/storage/db.py index 22e6fda..a4aa52a 100644 --- a/iaai_scraper/storage/db.py +++ b/openlane_scraper/storage/db.py @@ -11,7 +11,7 @@ from ..core.config import Settings from .models import Base, Car, Image, SyncRun from .schemas import CarRecord -logger = logging.getLogger("iaai_scraper.db") +logger = logging.getLogger("openlane_scraper.db") CAR_DB_FIELDS = { @@ -487,7 +487,7 @@ class PersistenceService: logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) return {"inserted": inserted, "updated": updated, "images_upserted": images_total} - def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int: + def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "openlane") -> int: """Помечает авто как проданные, если их нет в активном листинге (по origin_id).""" if not active_origin_ids: return 0 @@ -496,7 +496,7 @@ class PersistenceService: update(Car) .where(Car.origin_id.notin_(active_origin_ids)) .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like("iaai:%")) + .where(Car.origin_id.like("openlane:%")) .values(is_sold=True) ) result = session.execute(stmt) @@ -505,7 +505,7 @@ class PersistenceService: logger.info("Marked %d cars as sold (no longer in listing)", count) return count - def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int: + def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "openlane") -> int: """Помечает авто как проданные, если их URL нет в активном листинге. Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, @@ -543,7 +543,7 @@ class PersistenceService: LEFT JOIN _active_urls a ON c.origin_url = a.url WHERE a.url IS NULL AND c.is_sold = FALSE - AND c.origin_id LIKE 'iaai:%%' + AND c.origin_id LIKE 'openlane:%%' ) sub WHERE cars.id = sub.id """)) @@ -554,8 +554,8 @@ class PersistenceService: update(Car) .where(Car.origin_url.notin_(active_origin_urls)) .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like("iaai:%")) - .values(is_sold=True) + .where(Car.origin_id.like("openlane:%")) + .values(is_sold=True) ) result = session.execute(stmt) count = result.rowcount or 0 @@ -564,7 +564,7 @@ class PersistenceService: logger.info("Marked %d cars as sold by URL (no longer in listing)", count) return count - def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]: + def get_all_origin_ids_for_lane(self, prefix: str = "openlane:") -> set[str]: """Возвращает все известные origin_id для заданного префикса. Использует yield_per для потоковой загрузки при большом количестве записей. diff --git a/iaai_scraper/storage/enums.py b/openlane_scraper/storage/enums.py similarity index 97% rename from iaai_scraper/storage/enums.py rename to openlane_scraper/storage/enums.py index d25d8a5..253c73f 100644 --- a/iaai_scraper/storage/enums.py +++ b/openlane_scraper/storage/enums.py @@ -18,6 +18,7 @@ BODY_TYPE_ENUM_VALUES = ( ) COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") ORIGIN_ENUM_VALUES = ( + "OPENLANE", "IAAI", "NA", ) diff --git a/iaai_scraper/storage/models.py b/openlane_scraper/storage/models.py similarity index 100% rename from iaai_scraper/storage/models.py rename to openlane_scraper/storage/models.py diff --git a/iaai_scraper/storage/schemas.py b/openlane_scraper/storage/schemas.py similarity index 100% rename from iaai_scraper/storage/schemas.py rename to openlane_scraper/storage/schemas.py diff --git a/iaai_scraper/worker/__init__.py b/openlane_scraper/worker/__init__.py similarity index 100% rename from iaai_scraper/worker/__init__.py rename to openlane_scraper/worker/__init__.py diff --git a/iaai_scraper/worker/celery_app.py b/openlane_scraper/worker/celery_app.py similarity index 90% rename from iaai_scraper/worker/celery_app.py rename to openlane_scraper/worker/celery_app.py index 7ccabb1..f6d227a 100644 --- a/iaai_scraper/worker/celery_app.py +++ b/openlane_scraper/worker/celery_app.py @@ -9,8 +9,8 @@ from redis import Redis from ..core.config import settings from ..core.logs import setup_logging -logger = logging.getLogger("iaai_scraper.worker.celery_app") -STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched" +logger = logging.getLogger("openlane_scraper.worker.celery_app") +STARTUP_SYNC_DISPATCH_KEY = "openlane:state:startup_sync_dispatched" @celery_setup_logging.connect @@ -37,7 +37,7 @@ def _result_backend() -> str: celery_app = Celery( - "iaai_scraper", + "openlane_scraper", broker=_broker_url(), backend=_result_backend(), ) @@ -79,7 +79,7 @@ celery_app.conf.update( worker_hijack_root_logger=False, beat_schedule={ "periodic-sync-listing": { - "task": "iaai_scraper.worker.tasks.sync_listing_task", + "task": "openlane_scraper.worker.tasks.sync_listing_task", "schedule": settings.celery.beat_sync_interval_minutes * 60.0, "args": (), "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False}, @@ -87,11 +87,11 @@ celery_app.conf.update( } }, task_routes={ - "iaai_scraper.worker.tasks.*": {"queue": "scraping"} + "openlane_scraper.worker.tasks.*": {"queue": "scraping"} }, ) -celery_app.autodiscover_tasks(["iaai_scraper.worker"]) +celery_app.autodiscover_tasks(["openlane_scraper.worker"]) @worker_ready.connect @@ -118,7 +118,7 @@ def _on_worker_ready(**kwargs): logger.info("Worker ready — dispatching initial sync_listing task") celery_app.send_task( - "iaai_scraper.worker.tasks.sync_listing_task", + "openlane_scraper.worker.tasks.sync_listing_task", kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False}, queue="scraping", ) \ No newline at end of file diff --git a/openlane_scraper/worker/tasks.py b/openlane_scraper/worker/tasks.py new file mode 100644 index 0000000..6e62a05 --- /dev/null +++ b/openlane_scraper/worker/tasks.py @@ -0,0 +1,360 @@ +# Задачи Celery для синхронизации OpenLane marketplace → DB. + +import logging +import random +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from threading import Event, Thread + +from billiard.exceptions import SoftTimeLimitExceeded +from celery import shared_task +from redis import Redis + +from ..core.config import Settings +from ..scraper import OpenLaneScraper +from ..storage.db import PersistenceService + +logger = logging.getLogger("openlane_scraper.worker.tasks") + +SYNC_LISTING_LOCK_KEY = "openlane:locks:sync_listing" +SYNC_FULL_SCAN_DONE_KEY = "openlane:state:sync_full_scan_done" +SYNC_LISTING_TASK_NAME = "openlane_scraper.worker.tasks.sync_listing_task" + + +def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): + last_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return func() + except Exception as exc: + last_exc = exc + if attempt >= attempts: + break + delay = base_delay_s * (2 ** (attempt - 1)) + logger.warning( + "Operation failed (attempt %d/%d): %s. Retrying in %.1fs", + attempt, attempts, exc, delay, + ) + time.sleep(delay) + if last_exc is not None: + raise last_exc + + +def _run_browser_job(func, *args, **kwargs): + settings = Settings() + soft = settings.celery.task_soft_time_limit + hard = settings.celery.task_time_limit + wait_timeout = min(hard, soft + 120) if soft and hard else None + + executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="openlane-browser") + future = executor.submit(func, *args, **kwargs) + try: + result = future.result(timeout=wait_timeout) + except SoftTimeLimitExceeded: + future.cancel() + executor.shutdown(wait=False, cancel_futures=True) + raise + except TimeoutError: + future.cancel() + executor.shutdown(wait=False, cancel_futures=True) + raise SoftTimeLimitExceeded("Browser thread did not finish within time limit") + except Exception: + executor.shutdown(wait=False, cancel_futures=True) + raise + else: + executor.shutdown(wait=True) + return result + + +def _sync_listing_lock_ttl_seconds() -> int: + settings = Settings() + soft = settings.celery.task_soft_time_limit + hard = settings.celery.task_time_limit + effective_hard = min(hard, soft + 120) if soft else hard + return max(effective_hard + 120, 300) + + +def _get_persistence() -> PersistenceService: + settings = Settings() + persistence = PersistenceService(settings) + + def _ping_db() -> None: + with persistence.engine.connect() as conn: + conn.exec_driver_sql("SELECT 1") + + _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0) + return persistence + + +def _get_redis() -> Redis: + settings = Settings() + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + def _ping_redis() -> None: + redis_client.ping() + + _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0) + return redis_client + + +def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: + try: + acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + if acquired: + return True + ttl = redis_client.ttl(key) + if ttl is not None and ttl < 0: + logger.warning("Detected stale lock without TTL, removing: %s", key) + redis_client.delete(key) + return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + return False + except Exception: + logger.warning("Failed to acquire lock %s", key, exc_info=True) + return False + + +def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: + try: + refreshed = redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) + end + return 0 + """, + 1, key, owner_token, int(ttl_seconds), + ) + return bool(refreshed) + except Exception: + logger.warning("Failed to refresh lock %s", key, exc_info=True) + return None + + +def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: + try: + redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) + end + return 0 + """, + 1, key, owner_token, + ) + except Exception: + logger.warning("Failed to release lock %s", key, exc_info=True) + + +def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool: + try: + inspector = celery_app.control.inspect(timeout=1.0) + snapshots = [ + inspector.active() or {}, + inspector.reserved() or {}, + inspector.scheduled() or {}, + ] + except Exception: + logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True) + return True + + for snapshot in snapshots: + for entries in snapshot.values(): + for entry in entries or []: + task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") + if task_name != SYNC_LISTING_TASK_NAME: + continue + entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "") + if exclude_task_id and entry_id == exclude_task_id: + continue + return True + return False + + +def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool: + try: + owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY) + if not owner_token: + return False + except Exception: + logger.warning("Failed to read sync listing lock before cleanup", exc_info=True) + return False + + if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id): + logger.info("sync_listing lock preserved: active task still detected") + return False + + try: + ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY) + redis_client.delete(SYNC_LISTING_LOCK_KEY) + logger.warning( + "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart", + owner_token, ttl, + ) + return True + except Exception: + logger.warning("Failed to clear orphan sync listing lock", exc_info=True) + return False + + +def _is_full_scan_done(redis_client: Redis) -> bool: + try: + value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY) + except Exception: + logger.warning("Failed to read full scan state", exc_info=True) + return False + return str(value or "").strip() == "1" + + +def _set_full_scan_done(redis_client: Redis, done: bool) -> None: + try: + redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0") + except Exception: + logger.warning("Failed to persist full scan state", exc_info=True) + + +def _start_lock_heartbeat( + redis_client: Redis, key: str, owner_token: str, ttl_seconds: int, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) + + def _heartbeat() -> None: + while not stop_event.wait(interval_seconds): + refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) + if refreshed is False: + logger.warning("Lost sync_listing lock ownership for %s", owner_token) + return + + thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) + thread.start() + return stop_event, thread + + +@shared_task( + name="openlane_scraper.worker.tasks.sync_listing_task", + bind=True, + max_retries=3, + default_retry_delay=120, + acks_late=True, +) +def sync_listing_task( + self, + lane: str = "openlane_marketplace", + limit: int | None = None, + only_new: bool | None = None, + max_pages: int | None = None, + concurrency: int | None = None, +): + """Полный цикл синхронизации OpenLane marketplace → DB.""" + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + owner_token = f"{task_id}:{uuid.uuid4().hex}" + redis_client = _get_redis() + + lock_acquired = False + lock_ttl = _sync_listing_lock_ttl_seconds() + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id) + if orphan_cleared: + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + logger.info("sync_listing_task skipped: another sync is already running") + return { + "status": "skipped", + "reason": "sync_already_running", + "task_id": task_id, + } + + try: + full_scan_done = _is_full_scan_done(redis_client) + effective_only_new = False if not full_scan_done else only_new + + if not full_scan_done: + logger.info("Bootstrap mode: forcing full scan (only_new=False) until first complete run") + + # Рандомный jitter (0–120s) перед стартом. + jitter = random.uniform(0, 120) + logger.info("Anti-pattern jitter: waiting %.0fs before starting scrape", jitter) + time.sleep(jitter) + + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl, + ) + self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) + + def _job(): + with OpenLaneScraper() as scraper: + return scraper.sync_listing( + lane=lane, + limit=limit, + only_new=effective_only_new, + max_pages=max_pages, + concurrency=concurrency, + ) + + result = _run_browser_job(_job) + + if not full_scan_done: + if result.get("status") == "success": + _set_full_scan_done(redis_client, True) + logger.info("Bootstrap full scan completed; hourly schedule continues") + else: + _set_full_scan_done(redis_client, False) + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + } + logger.info( + "sync_listing_task completed: status=%s, %d upserted, %d failed", + summary["status"], summary["cars_upserted"], summary["cars_failed"], + ) + return summary + + except SoftTimeLimitExceeded: + logger.warning("sync_listing_task soft timeout exceeded — partial progress already saved to DB") + return { + "status": "timed_out", + "task_id": task_id, + "reason": "soft_time_limit_exceeded", + } + + except Exception as exc: + logger.error("sync_listing_task failed: %s", exc, exc_info=True) + try: + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + logger.error("sync_listing_task max retries exceeded, giving up") + return { + "status": "failed", + "task_id": task_id, + "error": str(exc), + } + finally: + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) + if lock_acquired: + _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) diff --git a/pyproject.toml b/pyproject.toml index 47a44fb..f5df0bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,9 @@ requires = ["setuptools>=68", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "iaai-scraper" +name = "openlane-scraper" version = "0.1.0" -description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL" +description = "OpenLane scraper service with FastAPI, Celery, Playwright and PostgreSQL" readme = "README.md" requires-python = ">=3.11" dependencies = [ @@ -31,13 +31,13 @@ dev = [ ] [project.scripts] -iaai = "iaai_scraper.cli:main" +openlane = "openlane_scraper.cli:main" [tool.setuptools] include-package-data = true [tool.setuptools.packages.find] -include = ["iaai_scraper*"] +include = ["openlane_scraper*"] [tool.pytest.ini_options] addopts = "-q --disable-warnings" diff --git a/runtime_config.json b/runtime_config.json index bb43bcb..8a79bdc 100644 --- a/runtime_config.json +++ b/runtime_config.json @@ -5,7 +5,7 @@ "ids_next_size": null, "ids_max_pages": null, "condition_check_enabled": false, - "lane": "iaai_cars", + "lane": "openlane_marketplace", "only_new": false, "limit": null }, diff --git a/scripts/explore_site.py b/scripts/explore_site.py new file mode 100644 index 0000000..10650ec --- /dev/null +++ b/scripts/explore_site.py @@ -0,0 +1,155 @@ +"""Open OpenLane in a real browser for manual exploration. + +Logs ALL network requests/responses so we can understand auth flow and API structure. +Usage: python scripts/explore_site.py +""" + +import json +import sys +from pathlib import Path +from datetime import datetime + +from playwright.sync_api import sync_playwright, Page, Request, Response + +LOG_DIR = Path("artifacts/explore") +LOG_DIR.mkdir(parents=True, exist_ok=True) +REQUESTS_LOG = LOG_DIR / f"requests_{datetime.now().strftime('%H%M%S')}.jsonl" + +captured = [] + + +def on_request(request: Request) -> None: + entry = { + "type": "request", + "ts": datetime.now().isoformat(), + "method": request.method, + "url": request.url, + "resource_type": request.resource_type, + "headers": dict(request.headers) if request.resource_type in ("xhr", "fetch", "document") else {}, + } + # Log API and auth requests to console + url = request.url.lower() + if any(k in url for k in ("/api/", "/auth", "/token", "/session", "/sign_in", "/login", "/oauth")): + print(f"\n>>> {request.method} {request.url}") + if request.post_data: + # Don't print passwords + post = request.post_data[:500] + if "password" in post.lower(): + post = "[CONTAINS PASSWORD - HIDDEN]" + print(f" body: {post}") + entry["post_data"] = post if "password" not in post.lower() else "[HIDDEN]" + captured.append(entry) + + +def on_response(response: Response) -> None: + url = response.url.lower() + if any(k in url for k in ("/api/", "/auth", "/token", "/session", "/sign_in", "/login", "/oauth")): + print(f"<<< {response.status} {response.url}") + entry = { + "type": "response", + "ts": datetime.now().isoformat(), + "status": response.status, + "url": response.url, + "headers": dict(response.headers), + } + # Try to capture response body for API/auth endpoints + try: + body = response.text() + if len(body) > 2000: + entry["body_preview"] = body[:2000] + "..." + else: + entry["body"] = body + # Print short preview + preview = body[:300] if len(body) > 300 else body + print(f" body: {preview}") + except Exception: + entry["body"] = "[could not read]" + captured.append(entry) + + +def save_log(): + with open(REQUESTS_LOG, "w", encoding="utf-8") as f: + for entry in captured: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + print(f"\n[*] Saved {len(captured)} entries to {REQUESTS_LOG}") + + +def main(): + print("=" * 60) + print("OpenLane Site Explorer") + print("=" * 60) + print("1. Browser will open at OpenLane sign_in page") + print("2. Log in manually") + print("3. Browse around — all API/auth requests are logged") + print("4. When done, close the browser or press Ctrl+C here") + print("=" * 60) + + with sync_playwright() as p: + browser = p.chromium.launch( + headless=False, + args=[ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + ], + ) + context = browser.new_context( + viewport={"width": 1920, "height": 1080}, + locale="en-US", + timezone_id="America/New_York", + user_agent=( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ), + ) + + page = context.new_page() + page.on("request", on_request) + page.on("response", on_response) + + print("\n[*] Opening https://app.openlane.com/sign_in ...") + page.goto("https://app.openlane.com/sign_in", wait_until="domcontentloaded") + + print("\n[*] Browser is open. Log in and explore the site.") + print("[*] I'm watching all network requests...") + print("[*] When you're done, just close the browser window.\n") + + try: + # Wait until the browser is closed by user + page.wait_for_event("close", timeout=0) + except KeyboardInterrupt: + print("\n[*] Interrupted by user") + except Exception: + pass + + # Capture cookies and storage state before closing + try: + cookies = context.cookies() + storage = context.storage_state() + + cookies_file = LOG_DIR / "cookies.json" + storage_file = LOG_DIR / "storage_state.json" + + with open(cookies_file, "w", encoding="utf-8") as f: + json.dump(cookies, f, indent=2, ensure_ascii=False) + with open(storage_file, "w", encoding="utf-8") as f: + json.dump(storage, f, indent=2, ensure_ascii=False) + + print(f"\n[*] Cookies saved to {cookies_file}") + print(f"[*] Storage state saved to {storage_file}") + + # Print cookie names + print(f"\n[*] Cookies ({len(cookies)}):") + for c in cookies: + print(f" {c['name']} = {c['value'][:30]}... (domain={c['domain']}, httpOnly={c.get('httpOnly', '?')})") + except Exception as e: + print(f"[!] Could not save state: {e}") + + browser.close() + + save_log() + print("\n[*] Done! Check artifacts/explore/ for logs.") + + +if __name__ == "__main__": + main() diff --git a/tests/test_db.py b/tests/test_db.py index 7720590..4a910e5 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -6,10 +6,10 @@ from pathlib import Path from sqlalchemy import select -from iaai_scraper.core.config import Settings -from iaai_scraper.storage.db import PersistenceService -from iaai_scraper.storage.models import Car, Image, SyncRun -from iaai_scraper.storage.schemas import CarRecord, ImageRecord +from openlane_scraper.core.config import Settings +from openlane_scraper.storage.db import PersistenceService +from openlane_scraper.storage.models import Car, Image, SyncRun +from openlane_scraper.storage.schemas import CarRecord, ImageRecord class TestPersistenceServiceIntegration(unittest.TestCase): @@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase): @staticmethod def _record(origin_id: str, *, price: int = 1000) -> CarRecord: return CarRecord( - parser_id=f"iaai:{origin_id}", + parser_id=f"openlane:{origin_id}", brand="Toyota", model="Camry", year=2014, price=price, - origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", - origin_id=origin_id, + origin_url=f"https://app.openlane.com/vehicles/{origin_id}", + origin_id=f"openlane:{origin_id}", slug=f"toyota-camry-{origin_id}", images=[ ImageRecord( - fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", - preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300", + fullres_image="https://images.openlane.com/full/1.jpg", + preview_image="https://images.openlane.com/thumb/1.jpg", order_index=0, ) ], @@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase): second = self._record("888") second.images = [ ImageRecord( - fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633", - preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + fullres_image="https://images.openlane.com/full/2.jpg", + preview_image="https://images.openlane.com/thumb/2.jpg", order_index=0, ) ] @@ -90,7 +90,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase): images = session.execute(select(Image)).scalars().all() self.assertEqual(len(images), 1) - self.assertIn("imageKeys=2", images[0].fullres_image) + self.assertIn("full/2", images[0].fullres_image) def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: first_run_id = self.persistence.start_sync_run("lane-a") @@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase): def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: first = self._record("OLD-ID") - first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + first.origin_url = "https://app.openlane.com/vehicles/45089484" self.persistence.upsert_car(first) second = self._record("NEW-ID") - second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + second.origin_url = "https://app.openlane.com/vehicles/45089484" result = self.persistence.upsert_car(second) self.assertEqual(result["action"], "updated") @@ -120,7 +120,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase): cars = session.execute(select(Car)).scalars().all() self.assertEqual(len(cars), 1) - self.assertEqual(cars[0].origin_id, "NEW-ID") + self.assertEqual(cars[0].origin_id, "openlane:NEW-ID") if __name__ == "__main__": diff --git a/tests/test_listing.py b/tests/test_listing.py deleted file mode 100644 index 7d8e991..0000000 --- a/tests/test_listing.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import unittest - -from iaai_scraper.browser.pace import HumanPacer -from iaai_scraper.core.config import Settings -from iaai_scraper.browser.listing import ListingCollector - - -class _FakePage: - def __init__(self, counts: dict[str, int]) -> None: - self._counts = counts - self._evaluate_result = False - self._evaluate_values: list[object] = [] - - class _Locator: - def __init__(self, count_value: int) -> None: - self._count_value = count_value - - def count(self) -> int: - return self._count_value - - def locator(self, selector: str) -> "_FakePage._Locator": - return _FakePage._Locator(self._counts.get(selector, 0)) - - def evaluate(self, _script: str): - if self._evaluate_values: - return self._evaluate_values.pop(0) - return self._evaluate_result - - -class TestListingUnit(unittest.TestCase): - def test_pagination_detection_and_page_number(self) -> None: - # Есть кнопка Next → True. - self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1}))) - # Нет селекторов → False. - self.assertFalse(ListingCollector._has_next_page(_FakePage({}))) - # Числовая пагинация через JS → True только если evaluate явно нашёл next. - page = _FakePage({}) - page._evaluate_result = True - self.assertTrue(ListingCollector._has_next_page(page)) - # Номер текущей страницы. - page2 = _FakePage({}) - page2._evaluate_values = [5] - self.assertEqual(ListingCollector._get_current_page_number(page2), 5) - - def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None: - collector = ListingCollector(Settings(), HumanPacer(Settings())) - html = """ -
-

Car 1

- -
- """ - - links = collector._extract_vehicle_links_from_html(html) - - self.assertEqual( - links, - [ - ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"), - ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"), - ], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_mapper.py b/tests/test_mapper.py new file mode 100644 index 0000000..990363b --- /dev/null +++ b/tests/test_mapper.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import unittest + +from openlane_scraper.openlane.mapper import ( + map_openlane_record, + map_openlane_records, + _parse_mileage, + _parse_engine_volume_cc, + _generate_slug, +) + + +class TestMapOpenLaneRecord(unittest.TestCase): + def _sample_record(self, **overrides) -> dict: + base = { + "id": "12345", + "make": "Toyota", + "model": "Camry", + "year": 2020, + "mileage": 45000, + "price": 18500, + "color": "White", + "body_type": "sedan", + "transmission": "automatic", + "drivetrain": "fwd", + "vin": "1HGBH41JXMN109186", + "url": "https://app.openlane.com/vehicles/12345", + "images": [ + {"full": "https://img.openlane.com/1_full.jpg", "thumbnail": "https://img.openlane.com/1_thumb.jpg"}, + {"full": "https://img.openlane.com/2_full.jpg", "thumbnail": "https://img.openlane.com/2_thumb.jpg"}, + ], + } + base.update(overrides) + return base + + def test_basic_mapping(self) -> None: + record = self._sample_record() + result = map_openlane_record(record) + self.assertIsNotNone(result) + self.assertEqual(result.brand, "TOYOTA") + self.assertEqual(result.model, "CAMRY") + self.assertEqual(result.year, 2020) + self.assertEqual(result.price, 18500) + self.assertEqual(result.mileage, 45000) + self.assertEqual(result.color, "white") + self.assertEqual(result.body_type, "SEDAN") + self.assertEqual(result.gearbox, "AT") + self.assertEqual(result.drive, "FWD") + self.assertEqual(result.origin, "OPENLANE") + self.assertEqual(result.origin_id, "openlane:12345") + self.assertEqual(result.origin_url, "https://app.openlane.com/vehicles/12345") + self.assertEqual(result.evaluation, "1HGBH41JXMN109186") + self.assertEqual(len(result.images), 2) + self.assertEqual(result.images[0].fullres_image, "https://img.openlane.com/1_full.jpg") + self.assertEqual(result.images[0].preview_image, "https://img.openlane.com/1_thumb.jpg") + + def test_missing_id_returns_none(self) -> None: + record = {"make": "Toyota", "model": "Camry"} + self.assertIsNone(map_openlane_record(record)) + + def test_missing_brand_returns_none(self) -> None: + record = {"id": "123", "model": "Camry"} + self.assertIsNone(map_openlane_record(record)) + + def test_missing_model_returns_none(self) -> None: + record = {"id": "123", "make": "Toyota"} + self.assertIsNone(map_openlane_record(record)) + + def test_vin_as_id_fallback(self) -> None: + record = self._sample_record() + del record["id"] + record["vin"] = "WBAPH5C52BA012345" + result = map_openlane_record(record) + self.assertIsNotNone(result) + self.assertEqual(result.origin_id, "openlane:WBAPH5C52BA012345") + + def test_existing_openlane_prefix_is_normalized(self) -> None: + record = self._sample_record(id="openlane:777") + result = map_openlane_record(record) + self.assertIsNotNone(result) + self.assertEqual(result.origin_id, "openlane:777") + + def test_whitespace_id_is_normalized(self) -> None: + record = self._sample_record(id=" 888 ") + result = map_openlane_record(record) + self.assertIsNotNone(result) + self.assertEqual(result.origin_id, "openlane:888") + + def test_image_url_strings(self) -> None: + record = self._sample_record(images=["https://a.com/1.jpg", "https://a.com/2.jpg"]) + result = map_openlane_record(record) + self.assertEqual(len(result.images), 2) + self.assertEqual(result.images[0].fullres_image, "https://a.com/1.jpg") + + def test_image_deduplication(self) -> None: + record = self._sample_record(images=[ + {"full": "https://a.com/same.jpg"}, + {"full": "https://a.com/same.jpg"}, + {"full": "https://a.com/other.jpg"}, + ]) + result = map_openlane_record(record) + self.assertEqual(len(result.images), 2) + + def test_vehicle_images_api_format(self) -> None: + record = self._sample_record(images=[ + { + "url": "https://a.com/original.jpg", + "large_resolution_url": "https://a.com/large.jpg", + "low_resolution_url": "https://a.com/medium.jpg", + } + ]) + result = map_openlane_record(record) + self.assertEqual(len(result.images), 1) + self.assertEqual(result.images[0].fullres_image, "https://a.com/original.jpg") + self.assertEqual(result.images[0].preview_image, "https://a.com/medium.jpg") + + def test_default_url_generation(self) -> None: + record = self._sample_record() + del record["url"] + result = map_openlane_record(record) + self.assertEqual(result.origin_url, "https://app.openlane.com/vehicles/12345") + + def test_nested_vehicle_fields(self) -> None: + record = { + "id": "999", + "vehicle": { + "make": "BMW", + "model": "X5", + "year": 2022, + "mileage": 10000, + }, + } + result = map_openlane_record(record) + self.assertIsNotNone(result) + self.assertEqual(result.brand, "BMW") + self.assertEqual(result.model, "X5") + self.assertEqual(result.year, 2022) + + def test_map_records_filters_invalid(self) -> None: + records = [ + self._sample_record(id="1"), + {"no_id": True}, + self._sample_record(id="2"), + ] + results = map_openlane_records(records) + self.assertEqual(len(results), 2) + + def test_sale_type_mapping(self) -> None: + record = self._sample_record(sale_type="buy_now") + result = map_openlane_record(record) + self.assertEqual(result.selling_type, "STOCK") + + record2 = self._sample_record(sale_type="tender") + result2 = map_openlane_record(record2) + self.assertEqual(result2.selling_type, "TENDER") + + +class TestParseMileage(unittest.TestCase): + def test_integer(self) -> None: + self.assertEqual(_parse_mileage(45000), 45000) + + def test_string_with_commas(self) -> None: + self.assertEqual(_parse_mileage("45,000 miles"), 45000) + + def test_none_returns_zero(self) -> None: + self.assertEqual(_parse_mileage(None), 0) + + def test_float(self) -> None: + self.assertEqual(_parse_mileage(45000.5), 45000) + + +class TestParseEngineVolume(unittest.TestCase): + def test_liters_float(self) -> None: + self.assertEqual(_parse_engine_volume_cc(2.5), 2500) + + def test_cc_int(self) -> None: + self.assertEqual(_parse_engine_volume_cc(2500), 2500) + + def test_string_liters(self) -> None: + self.assertEqual(_parse_engine_volume_cc("2.5L"), 2500) + + def test_none(self) -> None: + self.assertIsNone(_parse_engine_volume_cc(None)) + + +class TestGenerateSlug(unittest.TestCase): + def test_basic(self) -> None: + slug = _generate_slug(2020, "TOYOTA", "CAMRY", "openlane:123") + self.assertEqual(slug, "2020-toyota-camry-openlane-123") + + def test_no_year(self) -> None: + slug = _generate_slug(None, "BMW", "X5", "openlane:456") + self.assertEqual(slug, "bmw-x5-openlane-456") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py deleted file mode 100644 index 621a5fc..0000000 --- a/tests/test_mappers.py +++ /dev/null @@ -1,94 +0,0 @@ -from __future__ import annotations - -import unittest - -from iaai_scraper.parsing.mapper import CarMapper - - -class TestCarMapper(unittest.TestCase): - def setUp(self) -> None: - self.mapper = CarMapper() - - def test_deduplicates_images_by_image_key(self) -> None: - urls = [ - "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", - "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", - ] - - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/123~US", - vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, - payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, - ) - - self.assertEqual(len(record.images), 2) - self.assertIn("width=845", record.images[0].fullres_image) - self.assertIn("height=633", record.images[0].fullres_image) - - def test_no_damage_marker_is_not_damaged(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/123~US", - vehicle_summary={"make": "Ford", "model": "Focus"}, - payload_insights={ - "vehicle_core": {}, - "pricing": {}, - "damage": {"primary": "normal wear"}, - "auction": {}, - "images": {}, - }, - ) - - self.assertFalse(record.is_damaged) - - def test_unknown_empty_values_and_normalization(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/999~US", - vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, - payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, - ) - self.assertEqual(record.brand, "UNKNOWN") - self.assertEqual(record.model, "UNKNOWN") - self.assertEqual(record.drive, "NA") - self.assertEqual(record.gearbox, "NA") - - # Нормализация регистра и пробелов. - record2 = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/888~US", - vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, - payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, - ) - self.assertEqual(record2.drive, "FWD") - self.assertEqual(record2.gearbox, "AT") - - def test_price_and_currency_parsing(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/777~US", - vehicle_summary={"make": "Toyota", "model": "Corolla"}, - payload_insights={ - "vehicle_core": {}, - "pricing": {"buy_now": "USD 4,500 - 5,200"}, - "damage": {}, - "auction": {}, - "images": {}, - }, - ) - self.assertEqual(record.price, 5200) - - record2 = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/778~US", - vehicle_summary={"make": "Toyota", "model": "Corolla"}, - payload_insights={ - "vehicle_core": {}, - "pricing": {"buy_now": "€4.500,00"}, - "damage": {}, - "auction": {}, - "images": {}, - }, - ) - self.assertEqual(record2.currency, "EUR") - self.assertEqual(record2.price, 4500) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_openlane.py b/tests/test_openlane.py new file mode 100644 index 0000000..47690c8 --- /dev/null +++ b/tests/test_openlane.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from openlane_scraper.core.config import OpenLaneConfig +from openlane_scraper.openlane.checkpoint import OpenLaneCheckpoint, OpenLaneCheckpointStore +from openlane_scraper.openlane.client import OpenLaneClient +from openlane_scraper.openlane.writer import OpenLaneResultWriter + + +class TestOpenLaneCheckpointStore(unittest.TestCase): + def test_save_and_load_checkpoint(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + path = Path(tmp_dir) / "checkpoint.json" + store = OpenLaneCheckpointStore(path) + checkpoint = OpenLaneCheckpoint( + max_pages=512, + completed_pages=[1, 2, 3], + failed_pages=[7], + total_records=99, + last_saved_at="2026-04-20T00:00:00+00:00", + ) + + store.save(checkpoint) + loaded = store.load(max_pages=512) + + self.assertEqual(loaded.max_pages, 512) + self.assertEqual(loaded.completed_pages, [1, 2, 3]) + self.assertEqual(loaded.failed_pages, [7]) + self.assertEqual(loaded.total_records, 99) + self.assertEqual(loaded.last_saved_at, "2026-04-20T00:00:00+00:00") + + +class TestOpenLaneWriter(unittest.TestCase): + def test_append_and_finalize_outputs(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + jsonl_path = Path(tmp_dir) / "openlane.jsonl" + aggregated_path = Path(tmp_dir) / "openlane_aggregated.json" + writer = OpenLaneResultWriter(jsonl_path, aggregated_path) + + written = writer.append_page(1, [{"id": 1}, {"id": 2}]) + self.assertEqual(written, 2) + + summary = writer.finalize(max_pages=10, completed_pages=[1], failed_pages=[2]) + + self.assertEqual(summary["total_records"], 2) + self.assertEqual(summary["total_pages_completed"], 1) + self.assertEqual(summary["total_pages_failed"], 1) + self.assertTrue(aggregated_path.exists()) + + payload = json.loads(aggregated_path.read_text(encoding="utf-8")) + self.assertEqual(payload["items"][0]["page"], 1) + self.assertEqual(payload["items"][0]["record"]["id"], 1) + + +class TestOpenLaneConfig(unittest.TestCase): + def test_retry_schedule_defaults(self) -> None: + config = OpenLaneConfig() + self.assertGreaterEqual(len(config.retry_schedule_seconds), 1) + self.assertEqual(tuple(float(x) for x in config.retry_schedule_seconds), config.retry_schedule_seconds) + + def test_storage_state_defaults(self) -> None: + config = OpenLaneConfig() + self.assertTrue(config.storage_state_file) + self.assertIn("artifacts/openlane", config.storage_state_file) + self.assertIsInstance(config.persist_storage_state, bool) + + +class TestOpenLaneClient(unittest.TestCase): + def test_extract_records_from_common_shapes(self) -> None: + self.assertEqual( + OpenLaneClient._extract_records({"results": [{"id": 1}, {"id": 2}]}), + [{"id": 1}, {"id": 2}], + ) + self.assertEqual( + OpenLaneClient._extract_records({"data": {"items": [{"id": 3}]}}), + [{"id": 3}], + ) + self.assertEqual( + OpenLaneClient._extract_records({"vehicles": [{"id": 4}]}), + [{"id": 4}], + ) + self.assertEqual(OpenLaneClient._extract_records({"data": {"foo": "bar"}}), []) + + def test_extract_vehicle_images_from_common_shapes(self) -> None: + payload = { + "vehicle_images": [ + {"url": "https://img/1.png", "low_resolution_url": "https://img/1_small.png"}, + ] + } + self.assertEqual( + OpenLaneClient._extract_vehicle_images(payload), + [{"url": "https://img/1.png", "low_resolution_url": "https://img/1_small.png"}], + ) + + payload_nested = { + "data": { + "images": [ + {"url": "https://img/2.png"}, + ] + } + } + self.assertEqual( + OpenLaneClient._extract_vehicle_images(payload_nested), + [{"url": "https://img/2.png"}], + ) + + self.assertEqual(OpenLaneClient._extract_vehicle_images({"data": {"foo": 1}}), []) + + def test_extract_vehicle_images_batch_helper_normalization(self) -> None: + # Косвенно проверяем нормализацию входа под батчевый метод + ids = [" 123 ", "123", 456, "", None] + normalized = [] + seen = set() + for v in ids: + s = str(v).strip() + if not s or s in seen: + continue + seen.add(s) + normalized.append(s) + + self.assertEqual(normalized, ["123", "456", "None"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py deleted file mode 100644 index 72a2b6c..0000000 --- a/tests/test_parser.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -import unittest - -from iaai_scraper.parsing.parser import VehicleParser - - -class TestVehicleParserUnit(unittest.TestCase): - def setUp(self) -> None: - self.parser = VehicleParser() - - def test_parse_dom_pairs_and_title(self) -> None: - dom_text = """ - Stock #: - 45089484 - Primary Damage: - Front End - Odometer: - 50,123 mi (Actual) - """ - result = self.parser._parse_dom_key_value_pairs(dom_text) - self.assertEqual(result.get("lot_number"), "45089484") - self.assertEqual(result.get("primary_damage"), "Front End") - self.assertEqual(result.get("odometer"), "50,123 mi (Actual)") - - parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") - self.assertEqual(parsed["year"], "2014") - self.assertEqual(parsed["make"], "TOYOTA") - self.assertEqual(parsed["model"], "CAMRY") - - def test_extract_image_urls_filters_and_deduplicates(self) -> None: - vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" - payloads = [{"imageUrls": [ - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", - ]}] - urls = self.parser._extract_image_urls(payloads, "", vehicle_url) - self.assertEqual(len(urls), 1) - self.assertIn("45089484", urls[0]) - - def test_dom_hints_and_access_notes_detect_antibot(self) -> None: - hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") - self.assertTrue(hints["has_captcha_text"]) - self.assertTrue(hints["has_antibot_text"]) - - summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."} - notes = self.parser._build_access_notes(summary, []) - self.assertTrue(notes["possible_captcha"]) - self.assertTrue(notes["possible_antibot"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py deleted file mode 100644 index 3f4a379..0000000 --- a/tests/test_scraper.py +++ /dev/null @@ -1,273 +0,0 @@ -from __future__ import annotations - -import unittest -from types import SimpleNamespace -from unittest.mock import MagicMock - -from iaai_scraper.core.config import Settings -from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError -from iaai_scraper.scraper import IAAIScraper -from iaai_scraper.storage.schemas import CarRecord - - -def make_db_record(origin_id: str) -> dict[str, object]: - return CarRecord( - parser_id=f"iaai:{origin_id}", - brand="Toyota", - model="Camry", - origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", - origin_id=origin_id, - slug=f"toyota-camry-{origin_id}", - ).model_dump(mode="json") - - -class TestScraperSync(unittest.TestCase): - def _make_scraper(self) -> IAAIScraper: - s = Settings() - s.log_level = "CRITICAL" - s.database.url = "sqlite://" - return IAAIScraper(s) - - def test_sync_vehicle_uses_db_record(self) -> None: - scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=1) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) - scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) - - result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") - - self.assertEqual(result["status"], "success") - self.assertIn("trace_id", result) - scraper.persistence.upsert_car.assert_called_once() - - def test_only_new_routing_legacy_and_streaming(self) -> None: - # Legacy path: only_new без listing_url. - scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=2) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( - {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"}, - )) - scraper.collect_listing = MagicMock(return_value={ - "vehicle_urls": [ - "https://www.iaai.com/VehicleDetail/111~US", - "https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/VehicleDetail/333~US", - ] - }) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], - }) - result = scraper.sync_listing(only_new=True) - self.assertEqual(result["skipped_existing"], 2) - self.assertEqual(result["cars_upserted"], 1) - - # Streaming path: only_new + listing_url. - scraper2 = self._make_scraper() - scraper2.persistence.create_tables = MagicMock() - scraper2.persistence.start_sync_run = MagicMock(return_value=6) - scraper2.persistence.finish_sync_run = MagicMock() - scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used")) - scraper2._sync_listing_streaming = MagicMock(return_value={ - "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False}, - "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0, - "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(), - }) - result2 = scraper2.sync_listing( - only_new=True, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", - year_min=2020, year_max=2027, - ) - self.assertEqual(result2["cars_upserted"], 10) - scraper2._sync_listing_streaming.assert_called_once() - scraper2.collect_listing.assert_not_called() - - def test_segmented_sync_calls_per_segment_and_resumes(self) -> None: - scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=3) - scraper.persistence.finish_sync_run = MagicMock() - - call_args_log: list[dict] = [] - def _fake_sync_listing(**kwargs): - call_args_log.append(kwargs) - return { - "status": "success", "cars_upserted": 5, "cars_failed": 0, - "images_upserted": 10, "skipped_existing": 0, - "listing": {"vehicles_collected": 50}, "failures": [], - } - - scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing) - segments = [ - {"make": "TOYOTA", "year_min": 2020, "year_max": 2027}, - {"make": "FORD", "year_min": None, "year_max": None}, - {"make": "HONDA", "year_min": None, "year_max": None}, - ] - - # Resume: пропускаем TOYOTA, начинаем сразу с FORD. - result = scraper.sync_listing_segmented( - segments=segments, start_segment=1, - ) - - self.assertEqual(result["segments_completed"], 2) # FORD + HONDA - self.assertEqual(result["cars_upserted"], 10) - # URL содержит бренд. - self.assertIn("FORD", call_args_log[0]["listing_url"]) - self.assertIn("HONDA", call_args_log[1]["listing_url"]) - - def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None: - scraper = self._make_scraper() - scraper.sync_listing = MagicMock(side_effect=[ - { - "status": "failed", - "full_scan_completed": False, - "cars_upserted": 0, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "listing": {"vehicles_collected": 0}, - "failures": [{"vehicle_url": "segment", "error": "resume failed"}], - }, - { - "status": "success", - "full_scan_completed": True, - "cars_upserted": 1, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "listing": {"vehicles_collected": 10}, - "failures": [], - }, - ]) - - result = scraper.sync_listing_segmented( - segments=[ - {"make": "EAGLE", "year_min": None, "year_max": None}, - {"make": "FORD", "year_min": None, "year_max": None}, - ] - ) - - self.assertFalse(result["full_scan_completed"]) - self.assertEqual(result["status"], "partial_success") - - def test_build_segment_listing_url(self) -> None: - base = "https://www.iaai.com/Vehiclelisting/Cars" - self.assertEqual( - IAAIScraper._build_segment_listing_url(base, "TOYOTA"), - "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", - ) - self.assertEqual( - IAAIScraper._build_segment_listing_url(base, "LAND ROVER"), - "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER", - ) - self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base) - self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base) - - def test_guard_and_protection_detection(self) -> None: - with self.assertRaises(AntiBotDetectedError): - IAAIScraper._raise_if_blocked_or_incomplete( - {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, - "access_notes": {}, "vehicle_summary": {}}, - "https://www.iaai.com/VehicleDetail/999~US", - ) - with self.assertRaises(SiteStructureChangedError): - IAAIScraper._raise_if_blocked_or_incomplete( - {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, - "access_notes": {"possible_captcha": False, "possible_antibot": False}, - "vehicle_summary": {}}, - "https://www.iaai.com/VehicleDetail/999~US", - ) - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) - self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) - - def test_close_resets_browser_state(self) -> None: - scraper = self._make_scraper() - http_pool = MagicMock() - scraper._http_pool = http_pool - scraper.context = MagicMock() - scraper.browser = MagicMock() - scraper.playwright = MagicMock() - scraper.close() - http_pool.clear.assert_called_once() - self.assertIsNone(scraper.context) - self.assertIsNone(scraper.browser) - - def test_recover_empty_listing_page(self) -> None: - scraper = self._make_scraper() - page = MagicMock() - page_result = SimpleNamespace( - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")], - ) - scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) - scraper.listing_collector.open_cars_listing = MagicMock() - - # Страница > 1: reload. - _, urls = scraper._recover_empty_listing_page( - page, page_number=5, all_raw_urls=[], seen_urls=set(), - ) - page.reload.assert_called_once() - self.assertEqual(len(urls), 1) - - # Страница 1: переоткрытие листинга. - page.reset_mock() - _, urls = scraper._recover_empty_listing_page( - page, page_number=1, all_raw_urls=[], seen_urls=set(), - ) - scraper.listing_collector.open_cars_listing.assert_called_once() - page.reload.assert_not_called() - - def test_sync_listing_always_starts_from_page_one(self) -> None: - scraper = self._make_scraper() - scraper.settings.celery.batch_size = 1 - - page = MagicMock() - page_result = SimpleNamespace( - page_number=1, - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")], - next_page_detected=False, - ) - - scraper._get_page_with_warmup = MagicMock(return_value=page) - # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться. - scraper._reopen_listing_and_resume = MagicMock( - side_effect=AssertionError("pagination resume must not be used") - ) - scraper.listing_collector.open_cars_listing = MagicMock() - scraper.listing_collector.apply_filters = MagicMock(return_value={ - "make": None, - "model": None, - "year_min": None, - "year_max": None, - }) - scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) - scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"]) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, - "cars_failed": 0, - "images_upserted": 0, - "failures": [], - }) - - result = scraper._sync_listing_streaming( - make=None, - model=None, - lane="iaai_cars", - limit=None, - effective_only_new=False, - started_at=0.0, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE", - ) - - scraper.listing_collector.open_cars_listing.assert_called_once() - scraper._reopen_listing_and_resume.assert_not_called() - scraper.sync_batch.assert_called_once() - self.assertEqual(result["cars_upserted"], 1) - self.assertEqual(result["total"], 1) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_scraper_images.py b/tests/test_scraper_images.py new file mode 100644 index 0000000..7ed8cb7 --- /dev/null +++ b/tests/test_scraper_images.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import unittest + +from openlane_scraper.scraper import OpenLaneScraper +from openlane_scraper.storage.schemas import ImageRecord + + +class TestScraperImageHelpers(unittest.TestCase): + def test_build_image_records_supports_image_large_and_image(self) -> None: + payload = [ + { + "image": "https://img.cdn/vehicle-medium-1.jpg", + "image_large": "https://img.cdn/vehicle-large-1.jpg", + }, + { + "image": "https://img.cdn/vehicle-medium-2.jpg", + "large_resolution_url": "https://img.cdn/vehicle-large-2.jpg", + "low_resolution_url": "https://img.cdn/vehicle-low-2.jpg", + }, + ] + + records = OpenLaneScraper._build_image_records(payload) + + self.assertEqual(len(records), 2) + self.assertEqual(records[0].fullres_image, "https://img.cdn/vehicle-large-1.jpg") + self.assertEqual(records[0].preview_image, "https://img.cdn/vehicle-medium-1.jpg") + self.assertEqual(records[1].fullres_image, "https://img.cdn/vehicle-large-2.jpg") + self.assertEqual(records[1].preview_image, "https://img.cdn/vehicle-low-2.jpg") + + def test_merge_image_records_keeps_unique_and_reindexes(self) -> None: + existing = [ + ImageRecord( + fullres_image="https://img.cdn/1.jpg", + preview_image="https://img.cdn/1_small.jpg", + order_index=10, + ), + ImageRecord( + fullres_image="https://img.cdn/2.jpg", + preview_image="https://img.cdn/2_small.jpg", + order_index=11, + ), + ] + fetched = [ + ImageRecord( + fullres_image="https://img.cdn/2.jpg", + preview_image="https://img.cdn/2_small_new.jpg", + order_index=0, + ), + ImageRecord( + fullres_image="https://img.cdn/3.jpg", + preview_image="https://img.cdn/3_small.jpg", + order_index=1, + ), + ] + + merged = OpenLaneScraper._merge_image_records(existing, fetched) + + self.assertEqual([img.fullres_image for img in merged], [ + "https://img.cdn/2.jpg", + "https://img.cdn/3.jpg", + "https://img.cdn/1.jpg", + ]) + self.assertEqual([img.order_index for img in merged], [0, 1, 2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py index 2927c1e..9e4559b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,8 +2,7 @@ from __future__ import annotations import unittest -from iaai_scraper.core.utils import deep_find_key -from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS +from openlane_scraper.core.utils import deep_find_key class TestDeepFindKey(unittest.TestCase): @@ -21,55 +20,5 @@ class TestDeepFindKey(unittest.TestCase): self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"]) -class TestParseListingSegments(unittest.TestCase): - def test_empty_and_invalid_return_empty(self) -> None: - self.assertEqual(parse_listing_segments(""), []) - self.assertEqual(parse_listing_segments(" "), []) - self.assertEqual(parse_listing_segments("invalid"), []) - - def test_auto_segments_complete_coverage(self) -> None: - """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр.""" - segs = parse_listing_segments("auto") - - # Точное число сегментов. - expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES)) - self.assertEqual(len(segs), expected) - - # Все бренды из списка присутствуют. - makes_in_segments = {s["make"] for s in segs} - for make in IAAI_DEFAULT_MAKES: - self.assertIn(make, makes_in_segments) - - # Крупные бренды разбиты на 3 сегмента с годами. - toyota = [s for s in segs if s["make"] == "TOYOTA"] - self.assertEqual(len(toyota), 3) - for s in toyota: - self.assertIsNotNone(s["year_min"]) - - # Мелкие бренды без годов. - lexus = [s for s in segs if s["make"] == "LEXUS"] - self.assertEqual(len(lexus), 1) - self.assertIsNone(lexus[0]["year_min"]) - - # Годовые диапазоны покрывают 1950-2027. - years = set() - for yr_min, yr_max in _YEAR_SPLITS: - years.update(range(yr_min, yr_max + 1)) - for year in range(1950, 2027): - self.assertIn(year, years) - - def test_json_input_formats(self) -> None: - # Массив строк. - segs = parse_listing_segments('["toyota", "ford"]') - self.assertEqual(len(segs), 2) - self.assertEqual(segs[0]["make"], "TOYOTA") - - # Массив объектов с годами. - segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]') - self.assertEqual(segs[0]["make"], "BMW") - self.assertEqual(segs[0]["year_min"], 2020) - self.assertEqual(segs[0]["year_max"], 2025) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py index 2f8c088..3c00a5f 100644 --- a/tests/test_worker_tasks.py +++ b/tests/test_worker_tasks.py @@ -3,7 +3,7 @@ from __future__ import annotations import unittest from unittest.mock import MagicMock, patch -from iaai_scraper.worker import tasks +from openlane_scraper.worker import tasks class TestWorkerTaskLockHelpers(unittest.TestCase): @@ -52,8 +52,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks.sync_listing_task, "update_state"), \ + patch("openlane_scraper.worker.tasks.time.sleep"), \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 7, + "status": "success", "cars_upserted": 2, "cars_failed": 0, "images_upserted": 4, @@ -71,7 +73,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): tasks.sync_listing_task.push_request(id="task-123") try: - result = tasks.sync_listing_task.run(make="Toyota") + result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() @@ -117,8 +119,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks.sync_listing_task, "update_state"), \ + patch("openlane_scraper.worker.tasks.time.sleep"), \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 9, + "status": "success", "cars_upserted": 3, "cars_failed": 0, "images_upserted": 5, @@ -136,7 +140,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): tasks.sync_listing_task.push_request(id="task-456") try: - result = tasks.sync_listing_task.run(make="Honda") + result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() @@ -145,278 +149,41 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): self.assertEqual(acquire_lock.call_count, 2) release_lock.assert_called_once() - def test_checkpoint_save_load_roundtrip(self) -> None: - storage: dict[str, str] = {} - - def _fake_set(key, value, ex=None): - storage[key] = value - return True - - redis_client = MagicMock() - redis_client.set.side_effect = _fake_set - redis_client.get.side_effect = lambda key: storage.get(key) - - tasks._save_last_completed_segment(redis_client, 12) - - self.assertEqual(tasks._load_last_completed_segment(redis_client), 12) - self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS) - - def test_load_checkpoint_clears_invalid_payload(self) -> None: - redis_client = MagicMock() - redis_client.get.return_value = "{not-a-number" - - self.assertIsNone(tasks._load_last_completed_segment(redis_client)) - redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY) - - def test_load_checkpoint_empty_when_missing(self) -> None: - redis_client = MagicMock() - redis_client.get.return_value = None - - self.assertIsNone(tasks._load_last_completed_segment(redis_client)) - redis_client.delete.assert_not_called() - - def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None: - segments = [ - {"make": "ACURA"}, - {"make": "AUDI"}, - {"make": "BMW"}, - {"make": "EAGLE"}, - ] + def test_bootstrap_forces_full_scan(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_set_full_scan_done") as set_done, \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks, "_release_lock_if_owner"), \ patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.side_effect = lambda key: ( - "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None - ) - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - sync_segmented_mock = MagicMock(return_value={ - "run_id": 11, "status": "success", "full_scan_completed": True, - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, - "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], - }) - scraper_ctx = MagicMock() - scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__exit__.return_value = None - - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): - tasks.sync_listing_task.push_request(id="task-resume-seg") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW). - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2) - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) - release_lock.assert_called_once() - - def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None: - segments = [{"make": "ACURA"}, {"make": "AUDI"}] - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - # Оставшийся чекпоинт не должен использоваться. - redis_client.get.side_effect = lambda key: ( - "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None - ) - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - sync_segmented_mock = MagicMock(return_value={ - "run_id": 14, "status": "success", "full_scan_completed": True, - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, - "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], - }) - scraper_ctx = MagicMock() - scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__exit__.return_value = None - - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): - tasks.sync_listing_task.push_request(id="task-792") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) - self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"]) - clear_checkpoint.assert_called() - release_lock.assert_called_once() - - def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None: - segments = [{"make": "ACURA"}, {"make": "AUDI"}] - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=False), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.side_effect = lambda key: ( - "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None - ) - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - sync_segmented_mock = MagicMock(return_value={ - "run_id": 15, "status": "success", "full_scan_completed": True, - "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, - "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], - }) - scraper_ctx = MagicMock() - scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__exit__.return_value = None - - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): - tasks.sync_listing_task.push_request(id="task-beyond") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) - release_lock.assert_called_once() - - def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch("openlane_scraper.worker.tasks.time.sleep"), \ patch.object(tasks, "_run_browser_job", return_value={ - "run_id": 13, - "status": "partial_success", - "full_scan_completed": True, - "cars_upserted": 2, - "cars_failed": 1, - "images_upserted": 3, + "run_id": 11, + "status": "success", + "cars_upserted": 5, + "cars_failed": 0, + "images_upserted": 10, "skipped_existing": 0, - "elapsed_seconds": 4.0, - "failures": [{"vehicle_url": "v", "error": "e"}], - }), \ - patch.object(tasks.sync_listing_task, "update_state"): - get_persistence.return_value = MagicMock() + "elapsed_seconds": 3.0, + }): + persistence = MagicMock() + get_persistence.return_value = persistence redis_client = MagicMock() - redis_client.get.return_value = None get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) - tasks.sync_listing_task.push_request(id="task-791") + tasks.sync_listing_task.push_request(id="task-bootstrap") try: - result = tasks.sync_listing_task.run(make="Toyota") + result = tasks.sync_listing_task.run(only_new=True) finally: tasks.sync_listing_task.pop_request() - self.assertEqual(result["status"], "partial_success") - # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт - # до браузерного job + после. Главное — вызов произошёл. - clear_checkpoint.assert_called() - release_lock.assert_called_once() - - def test_try_set_followup_pending_deduplicates(self) -> None: - redis_client = MagicMock() - redis_client.set.side_effect = [True, False] - - self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) - self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) - - def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None: - redis_client = MagicMock() - redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT - - streak, should_enqueue = tasks._bump_bootstrap_failure_streak( - redis_client, - reason="max_retries_exceeded", - ) - - self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT) - self.assertFalse(should_enqueue) - redis_client.expire.assert_called_once_with( - tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, - tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, - ) - - def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None: - redis_client = MagicMock() - redis_client.incr.return_value = 1 - - streak, should_enqueue = tasks._bump_bootstrap_failure_streak( - redis_client, - reason="bootstrap_not_completed", - ) - - self.assertEqual(streak, 1) - self.assertTrue(should_enqueue) - - def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None: - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=False), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_try_set_followup_pending", return_value=True), \ - patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \ - patch.object(tasks, "_clear_followup_pending") as clear_pending, \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch.object(tasks, "_run_browser_job", return_value={ - "run_id": 99, - "status": "failed", - "full_scan_completed": False, - "cars_upserted": 0, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "elapsed_seconds": 1.0, - "failures": [{"vehicle_url": "listing", "error": "bad resume"}], - "listing": {"vehicles_collected": 0}, - }): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: - tasks.sync_listing_task.push_request(id="task-900") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "failed") - bump_streak.assert_called_once() - clear_pending.assert_called() - task_app.send_task.assert_not_called() + self.assertEqual(result["status"], "success") + # Bootstrap should set full_scan_done to True on success. + set_done.assert_called_once_with(redis_client, True) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/uv.lock b/uv.lock index e2820a9..747fb92 100644 --- a/uv.lock +++ b/uv.lock @@ -77,6 +77,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, ] +[[package]] +name = "brotli" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/16/c92ca344d646e71a43b8bb353f0a6490d7f6e06210f8554c8f874e454285/brotli-1.2.0.tar.gz", hash = "sha256:e310f77e41941c13340a95976fe66a8a95b01e783d430eeaf7a2f87e0a57dd0a", size = 7388632, upload-time = "2025-11-05T18:39:42.86Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/ef/f285668811a9e1ddb47a18cb0b437d5fc2760d537a2fe8a57875ad6f8448/brotli-1.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:15b33fe93cedc4caaff8a0bd1eb7e3dab1c61bb22a0bf5bdfdfd97cd7da79744", size = 863110, upload-time = "2025-11-05T18:38:12.978Z" }, + { url = "https://files.pythonhosted.org/packages/50/62/a3b77593587010c789a9d6eaa527c79e0848b7b860402cc64bc0bc28a86c/brotli-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:898be2be399c221d2671d29eed26b6b2713a02c2119168ed914e7d00ceadb56f", size = 445438, upload-time = "2025-11-05T18:38:14.208Z" }, + { url = "https://files.pythonhosted.org/packages/cd/e1/7fadd47f40ce5549dc44493877db40292277db373da5053aff181656e16e/brotli-1.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:350c8348f0e76fff0a0fd6c26755d2653863279d086d3aa2c290a6a7251135dd", size = 1534420, upload-time = "2025-11-05T18:38:15.111Z" }, + { url = "https://files.pythonhosted.org/packages/12/8b/1ed2f64054a5a008a4ccd2f271dbba7a5fb1a3067a99f5ceadedd4c1d5a7/brotli-1.2.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e1ad3fda65ae0d93fec742a128d72e145c9c7a99ee2fcd667785d99eb25a7fe", size = 1632619, upload-time = "2025-11-05T18:38:16.094Z" }, + { url = "https://files.pythonhosted.org/packages/89/5a/7071a621eb2d052d64efd5da2ef55ecdac7c3b0c6e4f9d519e9c66d987ef/brotli-1.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:40d918bce2b427a0c4ba189df7a006ac0c7277c180aee4617d99e9ccaaf59e6a", size = 1426014, upload-time = "2025-11-05T18:38:17.177Z" }, + { url = "https://files.pythonhosted.org/packages/26/6d/0971a8ea435af5156acaaccec1a505f981c9c80227633851f2810abd252a/brotli-1.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2a7f1d03727130fc875448b65b127a9ec5d06d19d0148e7554384229706f9d1b", size = 1489661, upload-time = "2025-11-05T18:38:18.41Z" }, + { url = "https://files.pythonhosted.org/packages/f3/75/c1baca8b4ec6c96a03ef8230fab2a785e35297632f402ebb1e78a1e39116/brotli-1.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9c79f57faa25d97900bfb119480806d783fba83cd09ee0b33c17623935b05fa3", size = 1599150, upload-time = "2025-11-05T18:38:19.792Z" }, + { url = "https://files.pythonhosted.org/packages/0d/1a/23fcfee1c324fd48a63d7ebf4bac3a4115bdb1b00e600f80f727d850b1ae/brotli-1.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:844a8ceb8483fefafc412f85c14f2aae2fb69567bf2a0de53cdb88b73e7c43ae", size = 1493505, upload-time = "2025-11-05T18:38:20.913Z" }, + { url = "https://files.pythonhosted.org/packages/36/e5/12904bbd36afeef53d45a84881a4810ae8810ad7e328a971ebbfd760a0b3/brotli-1.2.0-cp311-cp311-win32.whl", hash = "sha256:aa47441fa3026543513139cb8926a92a8e305ee9c71a6209ef7a97d91640ea03", size = 334451, upload-time = "2025-11-05T18:38:21.94Z" }, + { url = "https://files.pythonhosted.org/packages/02/8b/ecb5761b989629a4758c394b9301607a5880de61ee2ee5fe104b87149ebc/brotli-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:022426c9e99fd65d9475dce5c195526f04bb8be8907607e27e747893f6ee3e24", size = 369035, upload-time = "2025-11-05T18:38:22.941Z" }, + { url = "https://files.pythonhosted.org/packages/11/ee/b0a11ab2315c69bb9b45a2aaed022499c9c24a205c3a49c3513b541a7967/brotli-1.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:35d382625778834a7f3061b15423919aa03e4f5da34ac8e02c074e4b75ab4f84", size = 861543, upload-time = "2025-11-05T18:38:24.183Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2f/29c1459513cd35828e25531ebfcbf3e92a5e49f560b1777a9af7203eb46e/brotli-1.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7a61c06b334bd99bc5ae84f1eeb36bfe01400264b3c352f968c6e30a10f9d08b", size = 444288, upload-time = "2025-11-05T18:38:25.139Z" }, + { url = "https://files.pythonhosted.org/packages/3d/6f/feba03130d5fceadfa3a1bb102cb14650798c848b1df2a808356f939bb16/brotli-1.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acec55bb7c90f1dfc476126f9711a8e81c9af7fb617409a9ee2953115343f08d", size = 1528071, upload-time = "2025-11-05T18:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/2b/38/f3abb554eee089bd15471057ba85f47e53a44a462cfce265d9bf7088eb09/brotli-1.2.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:260d3692396e1895c5034f204f0db022c056f9e2ac841593a4cf9426e2a3faca", size = 1626913, upload-time = "2025-11-05T18:38:27.284Z" }, + { url = "https://files.pythonhosted.org/packages/03/a7/03aa61fbc3c5cbf99b44d158665f9b0dd3d8059be16c460208d9e385c837/brotli-1.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:072e7624b1fc4d601036ab3f4f27942ef772887e876beff0301d261210bca97f", size = 1419762, upload-time = "2025-11-05T18:38:28.295Z" }, + { url = "https://files.pythonhosted.org/packages/21/1b/0374a89ee27d152a5069c356c96b93afd1b94eae83f1e004b57eb6ce2f10/brotli-1.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adedc4a67e15327dfdd04884873c6d5a01d3e3b6f61406f99b1ed4865a2f6d28", size = 1484494, upload-time = "2025-11-05T18:38:29.29Z" }, + { url = "https://files.pythonhosted.org/packages/cf/57/69d4fe84a67aef4f524dcd075c6eee868d7850e85bf01d778a857d8dbe0a/brotli-1.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7a47ce5c2288702e09dc22a44d0ee6152f2c7eda97b3c8482d826a1f3cfc7da7", size = 1593302, upload-time = "2025-11-05T18:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/d5/3b/39e13ce78a8e9a621c5df3aeb5fd181fcc8caba8c48a194cd629771f6828/brotli-1.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:af43b8711a8264bb4e7d6d9a6d004c3a2019c04c01127a868709ec29962b6036", size = 1487913, upload-time = "2025-11-05T18:38:31.618Z" }, + { url = "https://files.pythonhosted.org/packages/62/28/4d00cb9bd76a6357a66fcd54b4b6d70288385584063f4b07884c1e7286ac/brotli-1.2.0-cp312-cp312-win32.whl", hash = "sha256:e99befa0b48f3cd293dafeacdd0d191804d105d279e0b387a32054c1180f3161", size = 334362, upload-time = "2025-11-05T18:38:32.939Z" }, + { url = "https://files.pythonhosted.org/packages/1c/4e/bc1dcac9498859d5e353c9b153627a3752868a9d5f05ce8dedd81a2354ab/brotli-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:b35c13ce241abdd44cb8ca70683f20c0c079728a36a996297adb5334adfc1c44", size = 369115, upload-time = "2025-11-05T18:38:33.765Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d4/4ad5432ac98c73096159d9ce7ffeb82d151c2ac84adcc6168e476bb54674/brotli-1.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9e5825ba2c9998375530504578fd4d5d1059d09621a02065d1b6bfc41a8e05ab", size = 861523, upload-time = "2025-11-05T18:38:34.67Z" }, + { url = "https://files.pythonhosted.org/packages/91/9f/9cc5bd03ee68a85dc4bc89114f7067c056a3c14b3d95f171918c088bf88d/brotli-1.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0cf8c3b8ba93d496b2fae778039e2f5ecc7cff99df84df337ca31d8f2252896c", size = 444289, upload-time = "2025-11-05T18:38:35.6Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b6/fe84227c56a865d16a6614e2c4722864b380cb14b13f3e6bef441e73a85a/brotli-1.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8565e3cdc1808b1a34714b553b262c5de5fbda202285782173ec137fd13709f", size = 1528076, upload-time = "2025-11-05T18:38:36.639Z" }, + { url = "https://files.pythonhosted.org/packages/55/de/de4ae0aaca06c790371cf6e7ee93a024f6b4bb0568727da8c3de112e726c/brotli-1.2.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:26e8d3ecb0ee458a9804f47f21b74845cc823fd1bb19f02272be70774f56e2a6", size = 1626880, upload-time = "2025-11-05T18:38:37.623Z" }, + { url = "https://files.pythonhosted.org/packages/5f/16/a1b22cbea436642e071adcaf8d4b350a2ad02f5e0ad0da879a1be16188a0/brotli-1.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67a91c5187e1eec76a61625c77a6c8c785650f5b576ca732bd33ef58b0dff49c", size = 1419737, upload-time = "2025-11-05T18:38:38.729Z" }, + { url = "https://files.pythonhosted.org/packages/46/63/c968a97cbb3bdbf7f974ef5a6ab467a2879b82afbc5ffb65b8acbb744f95/brotli-1.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4ecdb3b6dc36e6d6e14d3a1bdc6c1057c8cbf80db04031d566eb6080ce283a48", size = 1484440, upload-time = "2025-11-05T18:38:39.916Z" }, + { url = "https://files.pythonhosted.org/packages/06/9d/102c67ea5c9fc171f423e8399e585dabea29b5bc79b05572891e70013cdd/brotli-1.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3e1b35d56856f3ed326b140d3c6d9db91740f22e14b06e840fe4bb1923439a18", size = 1593313, upload-time = "2025-11-05T18:38:41.24Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/9526d14fa6b87bc827ba1755a8440e214ff90de03095cacd78a64abe2b7d/brotli-1.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:54a50a9dad16b32136b2241ddea9e4df159b41247b2ce6aac0b3276a66a8f1e5", size = 1487945, upload-time = "2025-11-05T18:38:42.277Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e8/3fe1ffed70cbef83c5236166acaed7bb9c766509b157854c80e2f766b38c/brotli-1.2.0-cp313-cp313-win32.whl", hash = "sha256:1b1d6a4efedd53671c793be6dd760fcf2107da3a52331ad9ea429edf0902f27a", size = 334368, upload-time = "2025-11-05T18:38:43.345Z" }, + { url = "https://files.pythonhosted.org/packages/ff/91/e739587be970a113b37b821eae8097aac5a48e5f0eca438c22e4c7dd8648/brotli-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:b63daa43d82f0cdabf98dee215b375b4058cce72871fd07934f179885aad16e8", size = 369116, upload-time = "2025-11-05T18:38:44.609Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/298c2ddf786bb7347a1cd71d63a347a79e5712a7c0cba9e3c3458ebd976f/brotli-1.2.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:6c12dad5cd04530323e723787ff762bac749a7b256a5bece32b2243dd5c27b21", size = 863080, upload-time = "2025-11-05T18:38:45.503Z" }, + { url = "https://files.pythonhosted.org/packages/84/0c/aac98e286ba66868b2b3b50338ffbd85a35c7122e9531a73a37a29763d38/brotli-1.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3219bd9e69868e57183316ee19c84e03e8f8b5a1d1f2667e1aa8c2f91cb061ac", size = 445453, upload-time = "2025-11-05T18:38:46.433Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f1/0ca1f3f99ae300372635ab3fe2f7a79fa335fee3d874fa7f9e68575e0e62/brotli-1.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:963a08f3bebd8b75ac57661045402da15991468a621f014be54e50f53a58d19e", size = 1528168, upload-time = "2025-11-05T18:38:47.371Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a6/2ebfc8f766d46df8d3e65b880a2e220732395e6d7dc312c1e1244b0f074a/brotli-1.2.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9322b9f8656782414b37e6af884146869d46ab85158201d82bab9abbcb971dc7", size = 1627098, upload-time = "2025-11-05T18:38:48.385Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2f/0976d5b097ff8a22163b10617f76b2557f15f0f39d6a0fe1f02b1a53e92b/brotli-1.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cf9cba6f5b78a2071ec6fb1e7bd39acf35071d90a81231d67e92d637776a6a63", size = 1419861, upload-time = "2025-11-05T18:38:49.372Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/d76df7176a2ce7616ff94c1fb72d307c9a30d2189fe877f3dd99af00ea5a/brotli-1.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7547369c4392b47d30a3467fe8c3330b4f2e0f7730e45e3103d7d636678a808b", size = 1484594, upload-time = "2025-11-05T18:38:50.655Z" }, + { url = "https://files.pythonhosted.org/packages/d3/93/14cf0b1216f43df5609f5b272050b0abd219e0b54ea80b47cef9867b45e7/brotli-1.2.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:fc1530af5c3c275b8524f2e24841cbe2599d74462455e9bae5109e9ff42e9361", size = 1593455, upload-time = "2025-11-05T18:38:51.624Z" }, + { url = "https://files.pythonhosted.org/packages/b3/73/3183c9e41ca755713bdf2cc1d0810df742c09484e2e1ddd693bee53877c1/brotli-1.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d2d085ded05278d1c7f65560aae97b3160aeb2ea2c0b3e26204856beccb60888", size = 1488164, upload-time = "2025-11-05T18:38:53.079Z" }, + { url = "https://files.pythonhosted.org/packages/64/6a/0c78d8f3a582859236482fd9fa86a65a60328a00983006bcf6d83b7b2253/brotli-1.2.0-cp314-cp314-win32.whl", hash = "sha256:832c115a020e463c2f67664560449a7bea26b0c1fdd690352addad6d0a08714d", size = 339280, upload-time = "2025-11-05T18:38:54.02Z" }, + { url = "https://files.pythonhosted.org/packages/f5/10/56978295c14794b2c12007b07f3e41ba26acda9257457d7085b0bb3bb90c/brotli-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e7c0af964e0b4e3412a0ebf341ea26ec767fa0b4cf81abb5e897c9338b5ad6a3", size = 375639, upload-time = "2025-11-05T18:38:55.67Z" }, +] + [[package]] name = "celery" version = "5.6.3" @@ -341,46 +389,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] -[[package]] -name = "iaai-scraper" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "alembic" }, - { name = "celery" }, - { name = "fastapi" }, - { name = "playwright" }, - { name = "psycopg2-binary" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "redis" }, - { name = "sqlalchemy" }, - { name = "uvicorn" }, -] - -[package.optional-dependencies] -dev = [ - { name = "pytest" }, - { name = "pytest-cov" }, -] - -[package.metadata] -requires-dist = [ - { name = "alembic", specifier = ">=1.14.0" }, - { name = "celery", specifier = ">=5.4.0" }, - { name = "fastapi", specifier = ">=0.115.0" }, - { name = "playwright", specifier = ">=1.53.0" }, - { name = "psycopg2-binary", specifier = ">=2.9.9" }, - { name = "pydantic", specifier = ">=2.8.2" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, - { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, - { name = "python-dotenv", specifier = ">=1.0.1" }, - { name = "redis", specifier = ">=5.2.0" }, - { name = "sqlalchemy", specifier = ">=2.0.32" }, - { name = "uvicorn", specifier = ">=0.34.0" }, -] -provides-extras = ["dev"] - [[package]] name = "idna" version = "3.11" @@ -500,6 +508,120 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "openlane-scraper" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "alembic" }, + { name = "brotli" }, + { name = "celery" }, + { name = "fastapi" }, + { name = "orjson" }, + { name = "playwright" }, + { name = "psycopg2-binary" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "redis" }, + { name = "sqlalchemy" }, + { name = "urllib3" }, + { name = "uvicorn" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", specifier = ">=1.14.0" }, + { name = "brotli", specifier = ">=1.1.0" }, + { name = "celery", specifier = ">=5.4.0" }, + { name = "fastapi", specifier = ">=0.115.0" }, + { name = "orjson", specifier = ">=3.10.0" }, + { name = "playwright", specifier = ">=1.53.0" }, + { name = "psycopg2-binary", specifier = ">=2.9.9" }, + { name = "pydantic", specifier = ">=2.8.2" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, + { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "redis", specifier = ">=5.2.0" }, + { name = "sqlalchemy", specifier = ">=2.0.32" }, + { name = "urllib3", specifier = ">=2.0.0" }, + { name = "uvicorn", specifier = ">=0.34.0" }, +] +provides-extras = ["dev"] + +[[package]] +name = "orjson" +version = "3.11.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/1b/2024d06792d0779f9dbc51531b61c24f76c75b9f4ce05e6f3377a1814cea/orjson-3.11.8.tar.gz", hash = "sha256:96163d9cdc5a202703e9ad1b9ae757d5f0ca62f4fa0cc93d1f27b0e180cc404e", size = 5603832, upload-time = "2026-03-31T16:16:27.878Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/41/5aa7fa3b0f4dc6b47dcafc3cea909299c37e40e9972feabc8b6a74e2730d/orjson-3.11.8-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:003646067cc48b7fcab2ae0c562491c9b5d2cbd43f1e5f16d98fd118c5522d34", size = 229229, upload-time = "2026-03-31T16:14:50.424Z" }, + { url = "https://files.pythonhosted.org/packages/0a/d7/57e7f2458e0a2c41694f39fc830030a13053a84f837a5b73423dca1f0938/orjson-3.11.8-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:ed193ce51d77a3830cad399a529cd4ef029968761f43ddc549e1bc62b40d88f8", size = 128871, upload-time = "2026-03-31T16:14:51.888Z" }, + { url = "https://files.pythonhosted.org/packages/53/4a/e0fdb9430983e6c46e0299559275025075568aad5d21dd606faee3703924/orjson-3.11.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f30491bc4f862aa15744b9738517454f1e46e56c972a2be87d70d727d5b2a8f8", size = 132104, upload-time = "2026-03-31T16:14:53.142Z" }, + { url = "https://files.pythonhosted.org/packages/08/4a/2025a60ff3f5c8522060cda46612d9b1efa653de66ed2908591d8d82f22d/orjson-3.11.8-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6eda5b8b6be91d3f26efb7dc6e5e68ee805bc5617f65a328587b35255f138bf4", size = 130483, upload-time = "2026-03-31T16:14:54.605Z" }, + { url = "https://files.pythonhosted.org/packages/2d/3c/b9cde05bdc7b2385c66014e0620627da638d3d04e4954416ab48c31196c5/orjson-3.11.8-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee8db7bfb6fe03581bbab54d7c4124a6dd6a7f4273a38f7267197890f094675f", size = 135481, upload-time = "2026-03-31T16:14:55.901Z" }, + { url = "https://files.pythonhosted.org/packages/ff/f2/a8238e7734de7cb589fed319857a8025d509c89dc52fdcc88f39c6d03d5a/orjson-3.11.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d8b5231de76c528a46b57010bbd83fb51e056aa0220a372fd5065e978406f1c", size = 146819, upload-time = "2026-03-31T16:14:57.548Z" }, + { url = "https://files.pythonhosted.org/packages/db/10/dbf1e2a3cafea673b1b4350e371877b759060d6018a998643b7040e5de48/orjson-3.11.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58a4a208a6fbfdb7a7327b8f201c6014f189f721fd55d047cafc4157af1bc62a", size = 132846, upload-time = "2026-03-31T16:14:58.91Z" }, + { url = "https://files.pythonhosted.org/packages/f8/fc/55e667ec9c85694038fcff00573d221b085d50777368ee3d77f38668bf3c/orjson-3.11.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f8952d6d2505c003e8f0224ff7858d341fa4e33fef82b91c4ff0ef070f2393c", size = 133580, upload-time = "2026-03-31T16:15:00.519Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a6/c08c589a9aad0cb46c4831d17de212a2b6901f9d976814321ff8e69e8785/orjson-3.11.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0022bb50f90da04b009ce32c512dc1885910daa7cb10b7b0cba4505b16db82a8", size = 142042, upload-time = "2026-03-31T16:15:01.906Z" }, + { url = "https://files.pythonhosted.org/packages/5c/cc/2f78ea241d52b717d2efc38878615fe80425bf2beb6e68c984dde257a766/orjson-3.11.8-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ff51f9d657d1afb6f410cb435792ce4e1fe427aab23d2fcd727a2876e21d4cb6", size = 423845, upload-time = "2026-03-31T16:15:03.703Z" }, + { url = "https://files.pythonhosted.org/packages/70/07/c17dcf05dd8045457538428a983bf1f1127928df5bf328cb24d2b7cddacb/orjson-3.11.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6dbe9a97bdb4d8d9d5367b52a7c32549bba70b2739c58ef74a6964a6d05ae054", size = 147729, upload-time = "2026-03-31T16:15:05.203Z" }, + { url = "https://files.pythonhosted.org/packages/90/6c/0fb6e8a24e682e0958d71711ae6f39110e4b9cd8cab1357e2a89cb8e1951/orjson-3.11.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a5c370674ebabe16c6ccac33ff80c62bf8a6e59439f5e9d40c1f5ab8fd2215b7", size = 136425, upload-time = "2026-03-31T16:15:07.052Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/4d3cc3a3d616035beb51b24a09bb872942dc452cf2df0c1d11ab35046d9f/orjson-3.11.8-cp311-cp311-win32.whl", hash = "sha256:0e32f7154299f42ae66f13488963269e5eccb8d588a65bc839ed986919fc9fac", size = 131870, upload-time = "2026-03-31T16:15:08.678Z" }, + { url = "https://files.pythonhosted.org/packages/13/26/9fe70f81d16b702f8c3a775e8731b50ad91d22dacd14c7599b60a0941cd1/orjson-3.11.8-cp311-cp311-win_amd64.whl", hash = "sha256:25e0c672a2e32348d2eb33057b41e754091f2835f87222e4675b796b92264f06", size = 127440, upload-time = "2026-03-31T16:15:09.994Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c6/b038339f4145efd2859c1ca53097a52c0bb9cbdd24f947ebe146da1ad067/orjson-3.11.8-cp311-cp311-win_arm64.whl", hash = "sha256:9185589c1f2a944c17e26c9925dcdbc2df061cc4a145395c57f0c51f9b5dbfcd", size = 127399, upload-time = "2026-03-31T16:15:11.412Z" }, + { url = "https://files.pythonhosted.org/packages/01/f6/8d58b32ab32d9215973a1688aebd098252ee8af1766c0e4e36e7831f0295/orjson-3.11.8-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:1cd0b77e77c95758f8e1100139844e99f3ccc87e71e6fc8e1c027e55807c549f", size = 229233, upload-time = "2026-03-31T16:15:12.762Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8b/2ffe35e71f6b92622e8ea4607bf33ecf7dfb51b3619dcfabfd36cbe2d0a5/orjson-3.11.8-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:6a3d159d5ffa0e3961f353c4b036540996bf8b9697ccc38261c0eac1fd3347a6", size = 128772, upload-time = "2026-03-31T16:15:14.237Z" }, + { url = "https://files.pythonhosted.org/packages/27/d2/1f8682ae50d5c6897a563cb96bc106da8c9cb5b7b6e81a52e4cc086679b9/orjson-3.11.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76070a76e9c5ae661e2d9848f216980d8d533e0f8143e6ed462807b242e3c5e8", size = 131946, upload-time = "2026-03-31T16:15:15.607Z" }, + { url = "https://files.pythonhosted.org/packages/52/4b/5500f76f0eece84226e0689cb48dcde081104c2fa6e2483d17ca13685ffb/orjson-3.11.8-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:54153d21520a71a4c82a0dbb4523e468941d549d221dc173de0f019678cf3813", size = 130368, upload-time = "2026-03-31T16:15:17.066Z" }, + { url = "https://files.pythonhosted.org/packages/da/4e/58b927e08fbe9840e6c920d9e299b051ea667463b1f39a56e668669f8508/orjson-3.11.8-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:469ac2125611b7c5741a0b3798cd9e5786cbad6345f9f400c77212be89563bec", size = 135540, upload-time = "2026-03-31T16:15:18.404Z" }, + { url = "https://files.pythonhosted.org/packages/56/7c/ba7cb871cba1bcd5cd02ee34f98d894c6cea96353ad87466e5aef2429c60/orjson-3.11.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14778ffd0f6896aa613951a7fbf4690229aa7a543cb2bfbe9f358e08aafa9546", size = 146877, upload-time = "2026-03-31T16:15:19.833Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5d/eb9c25fc1386696c6a342cd361c306452c75e0b55e86ad602dd4827a7fd7/orjson-3.11.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea56a955056a6d6c550cf18b3348656a9d9a4f02e2d0c02cabf3c73f1055d506", size = 132837, upload-time = "2026-03-31T16:15:21.282Z" }, + { url = "https://files.pythonhosted.org/packages/37/87/5ddeb7fc1fbd9004aeccab08426f34c81a5b4c25c7061281862b015fce2b/orjson-3.11.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53a0f57e59a530d18a142f4d4ba6dfc708dc5fdedce45e98ff06b44930a2a48f", size = 133624, upload-time = "2026-03-31T16:15:22.641Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/90048793db94ee4b2fcec4ac8e5ddb077367637d6650be896b3494b79bb7/orjson-3.11.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b48e274f8824567d74e2158199e269597edf00823a1b12b63d48462bbf5123e", size = 141904, upload-time = "2026-03-31T16:15:24.435Z" }, + { url = "https://files.pythonhosted.org/packages/c0/cf/eb284847487821a5d415e54149a6449ba9bfc5872ce63ab7be41b8ec401c/orjson-3.11.8-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:3f262401086a3960586af06c054609365e98407151f5ea24a62893a40d80dbbb", size = 423742, upload-time = "2026-03-31T16:15:26.155Z" }, + { url = "https://files.pythonhosted.org/packages/44/09/e12423d327071c851c13e76936f144a96adacfc037394dec35ac3fc8d1e8/orjson-3.11.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8e8c6218b614badf8e229b697865df4301afa74b791b6c9ade01d19a9953a942", size = 147806, upload-time = "2026-03-31T16:15:27.909Z" }, + { url = "https://files.pythonhosted.org/packages/b3/6d/37c2589ba864e582ffe7611643314785c6afb1f83c701654ef05daa8fcc7/orjson-3.11.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:093d489fa039ddade2db541097dbb484999fcc65fc2b0ff9819141e2ab364f25", size = 136485, upload-time = "2026-03-31T16:15:29.749Z" }, + { url = "https://files.pythonhosted.org/packages/be/c9/135194a02ab76b04ed9a10f68624b7ebd238bbe55548878b11ff15a0f352/orjson-3.11.8-cp312-cp312-win32.whl", hash = "sha256:e0950ed1bcb9893f4293fd5c5a7ee10934fbf82c4101c70be360db23ce24b7d2", size = 131966, upload-time = "2026-03-31T16:15:31.687Z" }, + { url = "https://files.pythonhosted.org/packages/ed/9a/9796f8fbe3cf30ce9cb696748dbb535e5c87be4bf4fe2e9ca498ef1fa8cf/orjson-3.11.8-cp312-cp312-win_amd64.whl", hash = "sha256:3cf17c141617b88ced4536b2135c552490f07799f6ad565948ea07bef0dcb9a6", size = 127441, upload-time = "2026-03-31T16:15:33.333Z" }, + { url = "https://files.pythonhosted.org/packages/cc/47/5aaf54524a7a4a0dd09dd778f3fa65dd2108290615b652e23d944152bc8e/orjson-3.11.8-cp312-cp312-win_arm64.whl", hash = "sha256:48854463b0572cc87dac7d981aa72ed8bf6deedc0511853dc76b8bbd5482d36d", size = 127364, upload-time = "2026-03-31T16:15:34.748Z" }, + { url = "https://files.pythonhosted.org/packages/66/7f/95fba509bb2305fab0073558f1e8c3a2ec4b2afe58ed9fcb7d3b8beafe94/orjson-3.11.8-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:3f23426851d98478c8970da5991f84784a76682213cd50eb73a1da56b95239dc", size = 229180, upload-time = "2026-03-31T16:15:36.426Z" }, + { url = "https://files.pythonhosted.org/packages/f6/9d/b237215c743ca073697d759b5503abd2cb8a0d7b9c9e21f524bcf176ab66/orjson-3.11.8-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ebaed4cef74a045b83e23537b52ef19a367c7e3f536751e355a2a394f8648559", size = 128754, upload-time = "2026-03-31T16:15:38.049Z" }, + { url = "https://files.pythonhosted.org/packages/42/3d/27d65b6d11e63f133781425f132807aef793ed25075fec686fc8e46dd528/orjson-3.11.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97c8f5d3b62380b70c36ffacb2a356b7c6becec86099b177f73851ba095ef623", size = 131877, upload-time = "2026-03-31T16:15:39.484Z" }, + { url = "https://files.pythonhosted.org/packages/dd/cc/faee30cd8f00421999e40ef0eba7332e3a625ce91a58200a2f52c7fef235/orjson-3.11.8-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:436c4922968a619fb7fef1ccd4b8b3a76c13b67d607073914d675026e911a65c", size = 130361, upload-time = "2026-03-31T16:15:41.274Z" }, + { url = "https://files.pythonhosted.org/packages/5c/bb/a6c55896197f97b6d4b4e7c7fd77e7235517c34f5d6ad5aadd43c54c6d7c/orjson-3.11.8-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ab359aff0436d80bfe8a23b46b5fea69f1e18aaf1760a709b4787f1318b317f", size = 135521, upload-time = "2026-03-31T16:15:42.758Z" }, + { url = "https://files.pythonhosted.org/packages/9c/7c/ca3a3525aa32ff636ebb1778e77e3587b016ab2edb1b618b36ba96f8f2c0/orjson-3.11.8-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f89b6d0b3a8d81e1929d3ab3d92bbc225688bd80a770c49432543928fe09ac55", size = 146862, upload-time = "2026-03-31T16:15:44.341Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0c/18a9d7f18b5edd37344d1fd5be17e94dc652c67826ab749c6e5948a78112/orjson-3.11.8-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29c009e7a2ca9ad0ed1376ce20dd692146a5d9fe4310848904b6b4fee5c5c137", size = 132847, upload-time = "2026-03-31T16:15:46.368Z" }, + { url = "https://files.pythonhosted.org/packages/23/91/7e722f352ad67ca573cee44de2a58fb810d0f4eb4e33276c6a557979fd8a/orjson-3.11.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:705b895b781b3e395c067129d8551655642dfe9437273211d5404e87ac752b53", size = 133637, upload-time = "2026-03-31T16:15:48.123Z" }, + { url = "https://files.pythonhosted.org/packages/af/04/32845ce13ac5bd1046ddb02ac9432ba856cc35f6d74dde95864fe0ad5523/orjson-3.11.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:88006eda83858a9fdf73985ce3804e885c2befb2f506c9a3723cdeb5a2880e3e", size = 141906, upload-time = "2026-03-31T16:15:49.626Z" }, + { url = "https://files.pythonhosted.org/packages/02/5e/c551387ddf2d7106d9039369862245c85738b828844d13b99ccb8d61fd06/orjson-3.11.8-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:55120759e61309af7fcf9e961c6f6af3dde5921cdb3ee863ef63fd9db126cae6", size = 423722, upload-time = "2026-03-31T16:15:51.176Z" }, + { url = "https://files.pythonhosted.org/packages/00/a3/ecfe62434096f8a794d4976728cb59bcfc4a643977f21c2040545d37eb4c/orjson-3.11.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:98bdc6cb889d19bed01de46e67574a2eab61f5cc6b768ed50e8ac68e9d6ffab6", size = 147801, upload-time = "2026-03-31T16:15:52.939Z" }, + { url = "https://files.pythonhosted.org/packages/18/6d/0dce10b9f6643fdc59d99333871a38fa5a769d8e2fc34a18e5d2bfdee900/orjson-3.11.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:708c95f925a43ab9f34625e45dcdadf09ec8a6e7b664a938f2f8d5650f6c090b", size = 136460, upload-time = "2026-03-31T16:15:54.431Z" }, + { url = "https://files.pythonhosted.org/packages/01/d6/6dde4f31842d87099238f1f07b459d24edc1a774d20687187443ab044191/orjson-3.11.8-cp313-cp313-win32.whl", hash = "sha256:01c4e5a6695dc09098f2e6468a251bc4671c50922d4d745aff1a0a33a0cf5b8d", size = 131956, upload-time = "2026-03-31T16:15:56.081Z" }, + { url = "https://files.pythonhosted.org/packages/c1/f9/4e494a56e013db957fb77186b818b916d4695b8fa2aa612364974160e91b/orjson-3.11.8-cp313-cp313-win_amd64.whl", hash = "sha256:c154a35dd1330707450bb4d4e7dd1f17fa6f42267a40c1e8a1daa5e13719b4b8", size = 127410, upload-time = "2026-03-31T16:15:57.54Z" }, + { url = "https://files.pythonhosted.org/packages/57/7f/803203d00d6edb6e9e7eef421d4e1adbb5ea973e40b3533f3cfd9aeb374e/orjson-3.11.8-cp313-cp313-win_arm64.whl", hash = "sha256:4861bde57f4d253ab041e374f44023460e60e71efaa121f3c5f0ed457c3a701e", size = 127338, upload-time = "2026-03-31T16:15:59.106Z" }, + { url = "https://files.pythonhosted.org/packages/6d/35/b01910c3d6b85dc882442afe5060cbf719c7d1fc85749294beda23d17873/orjson-3.11.8-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ec795530a73c269a55130498842aaa762e4a939f6ce481a7e986eeaa790e9da4", size = 229171, upload-time = "2026-03-31T16:16:00.651Z" }, + { url = "https://files.pythonhosted.org/packages/c2/56/c9ec97bd11240abef39b9e5d99a15462809c45f677420fd148a6c5e6295e/orjson-3.11.8-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:c492a0e011c0f9066e9ceaa896fbc5b068c54d365fea5f3444b697ee01bc8625", size = 128746, upload-time = "2026-03-31T16:16:02.673Z" }, + { url = "https://files.pythonhosted.org/packages/3b/e4/66d4f30a90de45e2f0cbd9623588e8ae71eef7679dbe2ae954ed6d66a41f/orjson-3.11.8-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:883206d55b1bd5f5679ad5e6ddd3d1a5e3cac5190482927fdb8c78fb699193b5", size = 131867, upload-time = "2026-03-31T16:16:04.342Z" }, + { url = "https://files.pythonhosted.org/packages/19/30/2a645fc9286b928675e43fa2a3a16fb7b6764aa78cc719dc82141e00f30b/orjson-3.11.8-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5774c1fdcc98b2259800b683b19599c133baeb11d60033e2095fd9d4667b82db", size = 124664, upload-time = "2026-03-31T16:16:05.837Z" }, + { url = "https://files.pythonhosted.org/packages/db/44/77b9a86d84a28d52ba3316d77737f6514e17118119ade3f91b639e859029/orjson-3.11.8-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7381c83dd3d4a6347e6635950aa448f54e7b8406a27c7ecb4a37e9f1ae08b", size = 129701, upload-time = "2026-03-31T16:16:07.407Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ea/eff3d9bfe47e9bc6969c9181c58d9f71237f923f9c86a2d2f490cd898c82/orjson-3.11.8-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14439063aebcb92401c11afc68ee4e407258d2752e62d748b6942dad20d2a70d", size = 141202, upload-time = "2026-03-31T16:16:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/52/c8/90d4b4c60c84d62068d0cf9e4d8f0a4e05e76971d133ac0c60d818d4db20/orjson-3.11.8-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa72e71977bff96567b0f500fc5bfd2fdf915f34052c782a4c6ebbdaa97aa858", size = 127194, upload-time = "2026-03-31T16:16:11.02Z" }, + { url = "https://files.pythonhosted.org/packages/8d/c7/ea9e08d1f0ba981adffb629811148b44774d935171e7b3d780ae43c4c254/orjson-3.11.8-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7679bc2f01bb0d219758f1a5f87bb7c8a81c0a186824a393b366876b4948e14f", size = 133639, upload-time = "2026-03-31T16:16:13.434Z" }, + { url = "https://files.pythonhosted.org/packages/6c/8c/ddbbfd6ba59453c8fc7fe1d0e5983895864e264c37481b2a791db635f046/orjson-3.11.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:14f7b8fcb35ef403b42fa5ecfa4ed032332a91f3dc7368fbce4184d59e1eae0d", size = 141914, upload-time = "2026-03-31T16:16:14.955Z" }, + { url = "https://files.pythonhosted.org/packages/4e/31/dbfbefec9df060d34ef4962cd0afcb6fa7a9ec65884cb78f04a7859526c3/orjson-3.11.8-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c2bdf7b2facc80b5e34f48a2d557727d5c5c57a8a450de122ae81fa26a81c1bc", size = 423800, upload-time = "2026-03-31T16:16:16.594Z" }, + { url = "https://files.pythonhosted.org/packages/87/cf/f74e9ae9803d4ab46b163494adba636c6d7ea955af5cc23b8aaa94cfd528/orjson-3.11.8-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ccd7ba1b0605813a0715171d39ec4c314cb97a9c85893c2c5c0c3a3729df38bf", size = 147837, upload-time = "2026-03-31T16:16:18.585Z" }, + { url = "https://files.pythonhosted.org/packages/64/e6/9214f017b5db85e84e68602792f742e5dc5249e963503d1b356bee611e01/orjson-3.11.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cdbc8c9c02463fef4d3c53a9ba3336d05496ec8e1f1c53326a1e4acc11f5c600", size = 136441, upload-time = "2026-03-31T16:16:20.151Z" }, + { url = "https://files.pythonhosted.org/packages/24/dd/3590348818f58f837a75fb969b04cdf187ae197e14d60b5e5a794a38b79d/orjson-3.11.8-cp314-cp314-win32.whl", hash = "sha256:0b57f67710a8cd459e4e54eb96d5f77f3624eba0c661ba19a525807e42eccade", size = 131983, upload-time = "2026-03-31T16:16:21.823Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0f/b6cb692116e05d058f31ceee819c70f097fa9167c82f67fabe7516289abc/orjson-3.11.8-cp314-cp314-win_amd64.whl", hash = "sha256:735e2262363dcbe05c35e3a8869898022af78f89dde9e256924dc02e99fe69ca", size = 127396, upload-time = "2026-03-31T16:16:23.685Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d1/facb5b5051fabb0ef9d26c6544d87ef19a939a9a001198655d0d891062dd/orjson-3.11.8-cp314-cp314-win_arm64.whl", hash = "sha256:6ccdea2c213cf9f3d9490cbd5d427693c870753df41e6cb375bd79bcbafc8817", size = 127330, upload-time = "2026-03-31T16:16:25.496Z" }, +] + [[package]] name = "packaging" version = "26.0" @@ -968,6 +1090,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, ] +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] + [[package]] name = "uvicorn" version = "0.44.0"