qananasikq/Openlane
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial openlane project

Browse Source
This commit is contained in:
qananasikq
2026-04-21 23:01:33 +03:00
parent 55203d33ea
commit cccf6b3916
70 changed files with 4176 additions and 6804 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
.env.example
View File

@@ -1,52 +1,47 @@
IAAI_HEADLESS=true
IAAI_LOG_LEVEL=INFO
# --- General ---
OPENLANE_HEADLESS=true
OPENLANE_LOG_LEVEL=INFO
OPENLANE_TIMEOUT_MS=45000
IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
IAAI_MAX_CAPTURED_REQUESTS=40
IAAI_MAX_CAPTURED_JSON_RESPONSES=20
# --- OpenLane Auth & API ---
OPENLANE_USERNAME=
OPENLANE_PASSWORD=
OPENLANE_SIGN_IN_URL=https://app.openlane.com/sign_in
OPENLANE_SEARCH_URL=https://app.openlane.com/search?tab=marketplace
OPENLANE_API_SEARCH_URL=https://app.openlane.com/api/v4/search
OPENLANE_SOURCE_TAB=marketplace
OPENLANE_SALE_TYPES=marketplace
OPENLANE_PAGE_SIZE=30
OPENLANE_MAX_PAGES=512
OPENLANE_MAX_CARS_LIMIT=20
OPENLANE_THROTTLE_MIN_SECONDS=0.3
OPENLANE_THROTTLE_MAX_SECONDS=0.8
OPENLANE_RETRY_SCHEDULE_SECONDS=2,5,10
OPENLANE_REQUEST_TIMEOUT_MS=45000
OPENLANE_REFRESH_TOKEN=
OPENLANE_OKTA_CLIENT_ID=
OPENLANE_OKTA_TOKEN_ENDPOINT=https://okta.iam.karglobal.com/oauth2/default/v1/token
OPENLANE_OKTA_REDIRECT_URI=https://app.openlane.com/sign_in
OPENLANE_OKTA_TIMEOUT_SECONDS=15
IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
IAAI_LISTING_SEGMENTS=auto
IAAI_MAX_PAGES_PER_RUN=999999
IAAI_MAX_VEHICLES_PER_RUN=999999
IAAI_PAGE_LINK_LIMIT=999999
IAAI_INCLUDE_PAGINATION=true
IAAI_COLLECT_CURRENT_PAGE_ONLY=false
# --- Output & State ---
OPENLANE_OUTPUT_DIR=artifacts/openlane
OPENLANE_JSONL_OUTPUT=artifacts/openlane/openlane_search.jsonl
OPENLANE_AGGREGATED_OUTPUT=artifacts/openlane/openlane_search_aggregated.json
OPENLANE_CHECKPOINT_FILE=artifacts/openlane/openlane_checkpoint.json
OPENLANE_STORAGE_STATE_FILE=/data/openlane/openlane_storage_state.json
OPENLANE_PERSIST_STORAGE_STATE=true
IAAI_HUMAN_PACE_ENABLED=true
IAAI_PARALLEL_TABS=10
CELERY_BATCH_SIZE=100
IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
IAAI_BETWEEN_VEHICLES_MIN_S=0.02
IAAI_BETWEEN_VEHICLES_MAX_S=0.08
IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
# --- Database (PostgreSQL) ---
OPENLANE_DATABASE_URL=postgresql+psycopg2://openlane:openlane@postgres:5432/openlane_scraper
OPENLANE_DATABASE_ECHO=false
OPENLANE_DATABASE_POOL_SIZE=5
OPENLANE_DATABASE_MAX_OVERFLOW=5
IAAI_SYNC_ONLY_NEW=false
IAAI_TOKENS_FILE=/data/tokens.json
IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
IAAI_MAX_RETRIES=5
IAAI_RETRY_DELAY_SECONDS=4
IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
IAAI_RETRY_JITTER_SECONDS=0.5
IAAI_TIMEOUT_MS=90000
IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
IAAI_DATABASE_ECHO=false
IAAI_DATABASE_POOL_SIZE=5
IAAI_DATABASE_MAX_OVERFLOW=5
IAAI_REDIS_URL=redis://redis:6379/0
# --- Redis ---
OPENLANE_REDIS_URL=redis://redis:6379/0
# --- Celery ---
CELERY_BROKER_URL=redis://redis:6379/0
CELERY_RESULT_BACKEND=redis://redis:6379/0
CELERY_TASK_SOFT_TIME_LIMIT=86400
@@ -55,3 +50,13 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000
CELERY_WORKER_CONCURRENCY=1
CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
CELERY_BEAT_SYNC_LIMIT=0
# --- Docker host ports (override if occupied) ---
OPENLANE_API_PORT=58000
OPENLANE_POSTGRES_PORT=55432
OPENLANE_REDIS_PORT=56379
# --- Proxy (optional) ---
# OPENLANE_PROXY_SERVER=http://127.0.0.1:8899
# OPENLANE_PROXY_USERNAME=
# OPENLANE_PROXY_PASSWORD=

10
Dockerfile
View File

@@ -13,17 +13,15 @@ RUN pip install --no-cache-dir uv \
&& pip install --no-cache-dir -r /tmp/requirements.txt \
&& pip install --no-cache-dir playwright-stealth
# Speed-up libs: orjson (fast JSON parse), brotli (HTTP br decompress).
# Pinned outside uv.lock to avoid lock-regen churn.
# orjson + brotli для ускорения JSON/HTTP.
RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0"
# Install both Chromium and Firefox. Chromium is the default engine in Docker
# because it works more reliably with the current IAAI listing page.
# Chromium + Firefox.
RUN python -m playwright install chromium firefox
COPY . .
RUN pip install --no-cache-dir -e .
RUN python -m compileall -q iaai_scraper
RUN python -m compileall -q openlane_scraper
RUN chmod +x entrypoint.sh
RUN chown -R app:app /app
@@ -33,4 +31,4 @@ USER app
# По умолчанию API, но worker/beat переопределяют CMD в docker-compose
ENTRYPOINT ["./entrypoint.sh"]
CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
CMD ["uvicorn", "openlane_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

314
README.md
View File

@@ -1,231 +1,138 @@
# IAAI Scraper
# OpenLane Scraper
Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.
Парсер автомобилей с [OpenLane marketplace](https://app.openlane.com). Забирает данные через API (`GET /api/v4/search`), сохраняет в PostgreSQL. Работает через Playwright (для авторизации), FastAPI, Celery и Docker.
## Как устроен сайт и его защита
## Архитектура
У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.
- **Авторизация**: Okta OAuth2 PKCE через `okta.iam.karglobal.com`. Сессия сохраняется в `storage_state.json` и переиспользуется между запусками.
- **Сбор данных**: API-запросы с заголовками `x-rbz-user-id`, `x-rbz-dealership-id` (извлекаются из JWT access_token).
- **Защита аккаунта**: throttle 2–5s между запросами, jitter 0–120s перед стартом, circuit breaker (2 ошибки подряд → стоп), 403→немедленный стоп, 429→backoff×3.
- **Anti-detection**: блокировка ресурсов (bugsnag, intercom, pusher, analytics), stealth patches.
- **Anti-bot** — headless Chromium без прокси часто режется.
- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
- **Cookie consent** — при первом заходе показывают баннер.
Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.
## Локальный запуск (без Docker)
## Локальный запуск
### Требования
- **Python 3.11+**
- **Git**
Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
- **PostgreSQL** и **Redis** (для Celery)
### Быстрый старт
```bash
git clone <repo-url>
cd iaai_scraper_project
cd openlane_scraper_project
pip install -e .
playwright install firefox
iaai init-db
```
playwright install chromium
`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker).
cp .env.example .env
# Заполнить OPENLANE_USERNAME, OPENLANE_PASSWORD в .env
Готовый `.env` уже в репозитории и настроен на SQLite ничего менять не нужно.
### Запуск парсера
**Скрапинг одной машины:**
```bash
iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
```
**Сбор листинга + скрапинг (например Toyota, 10 штук):**
```bash
iaai sync-listing --make Toyota --limit 10
```
**Только ссылки с листинга (без скрапинга):**
```bash
iaai collect-listing --make Toyota
```
**С видимым браузером (для отладки):**
```bash
iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
```
**Проверка, что записалось в базу (`iaai_scraper.db`):**
```bash
python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
```
Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
### Полный стек (API + Worker + Beat)
Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
```env
IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
IAAI_REDIS_URL=redis://localhost:6379/0
CELERY_BROKER_URL=redis://localhost:6379/0
CELERY_RESULT_BACKEND=redis://localhost:6379/0
```
Применить миграции и запустить в трёх терминалах:
```bash
alembic upgrade head
```
### Запуск
```bash
# Терминал 1 — API
uvicorn iaai_scraper.api.app:app --reload --port 8000
uvicorn openlane_scraper.api.app:app --reload --port 8000
# Терминал 2 — Celery Worker
celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
celery -A openlane_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
# Терминал 3 — Celery Beat (периодический запуск)
celery -A iaai_scraper.worker.celery_app beat --loglevel=info
# Терминал 3 — Celery Beat (каждый час)
celery -A openlane_scraper.worker.celery_app beat --loglevel=info
```
API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
CLI:
---
```bash
openlane scrape-openlane --limit 20
```
## Запуск через Docker (все сервисы в контейнерах)
## Docker
```bash
cp .env.example .env
docker compose up -d
```
Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
Сервисы: `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`.
В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
- `beat` запускает сбор **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`)
- Лимит по умолчанию: 20 машин (`OPENLANE_MAX_CARS_LIMIT=20`)
По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
API (Docker): `http://localhost:58000` · Swagger: `http://localhost:58000/docs`
Swagger-документация: `http://localhost:8000/docs`
### Первый логин (storage state)
```bash
docker compose ps
docker compose run --rm --service-ports openlane-auth openlane openlane-login
# Войти вручную в браузере — storage state сохранится в volume
```
- `api` имеет healthcheck на `GET /health`
- `worker` имеет healthcheck через `celery inspect ping`
- `beat` имеет healthcheck по файлу `celerybeat-schedule`
## API
**Здоровье и статистика:**
- `GET /health` — статус сервиса и подключения к БД
- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
**Машины:**
- `GET /health` — статус сервиса
- `GET /api/v1/stats` — статистика (машины, картинки, топ брендов)
- `GET /api/v1/cars` — список с пагинацией
- `GET /api/v1/cars/{id}` — карточка с картинками
- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number
**Задачи:**
- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
- `POST /api/v1/tasks/sync-listing` — запустить сбор
- `GET /api/v1/sync-runs` — история запусков
Скрапим конкретную машину:
```bash
curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
-H "Content-Type: application/json" \
-d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
```
## CLI
Для отладки без API и Celery:
```bash
iaai init-db
iaai collect-listing --make Toyota
iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
iaai sync-listing --limit 10
```
`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic.
## Структура
```text
iaai_scraper/
scraper.py — оркестратор: связывает browser → parser → storage
cli.py — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
proxy_bridge.py — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией)
openlane_scraper/
scraper.py — оркестратор: circuit breaker, batched upsert, early-stop
cli.py — CLI-команды (scrape-openlane, openlane-login)
browser/
factory.py — создание браузера, stealth-инъекции, fingerprint
listing.py — сбор ссылок на машины с листинга, пагинация
network.py — перехват XHR/fetch ответов через Playwright events
pace.py — рандомные паузы и движение мыши
factory.py — создание браузера, stealth-инъекции, resource blocking
parsing/
parser.py — VehicleParser: DOM + XHR JSON + embedded JSON
mapper.py — CarMapper: нормализация → CarRecord
openlane/
auth.py — авторизация через storage_state / Okta PKCE
client.py — запросы к /api/v4/search с JWT-заголовками
checkpoint.py — checkpoint/resume по страницам
writer.py — JSONL + aggregated JSON
runner.py — orchestration, retry/backoff, progress
storage/
models.py — SQLAlchemy: Car, Image, SyncRun
schemas.py — Pydantic: CarRecord, ImageRecord, CarRead
enums.py — допустимые значения (drive, gearbox, body_type, ...)
enums.py — enum-значения (drive, gearbox, body_type, ...)
db.py — PersistenceService: upsert, sync runs, статистика
api/
app.py — FastAPI factory, lifespan, роутеры
deps.py — dependency injection (Settings, PersistenceService)
routes/
health.py — GET /health
cars.py — CRUD по машинам + GET /stats
tasks.py — запуск Celery-задач и история sync-runs
app.py — FastAPI factory
deps.py — dependency injection
routes/ — health, cars, tasks
worker/
celery_app.py — конфиг Celery, beat-расписание
tasks.py — sync_vehicle_task, sync_listing_task
tasks.py — sync_listing_task с jitter и lock
core/
config.py — Settings (dataclass), env-переменные
logs.py — логирование с trace_id (ContextVar)
retry.py — декоратор @retryable с exponential backoff
runtime_config.py — чтение и нормализация runtime-конфига
utils.py — VIN_RE, deep_find_key, вспомогательные функции
alembic/ — миграции БД
tests/ — тесты
logs.py — логирование с trace_id
retry.py — @retryable с exponential backoff
runtime_config.py — runtime-конфиг
utils.py — утилиты
```
## Конфигурация
Всё через env-переменные (полный список в `.env.example`):
- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
- **Redis:** `IAAI_REDIS_URL`
- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
- **OpenLane**: `OPENLANE_USERNAME`, `OPENLANE_PASSWORD`, `OPENLANE_MAX_CARS_LIMIT`
- **БД**: `OPENLANE_DATABASE_URL`, `OPENLANE_DATABASE_POOL_SIZE`
- **Redis**: `OPENLANE_REDIS_URL`
- **Celery**: `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
- **Прокси**: `OPENLANE_PROXY_SERVER`, `OPENLANE_PROXY_USERNAME`, `OPENLANE_PROXY_PASSWORD`
## Миграции
В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.
Вручную:
В Docker миграции выполняются сервисом `migrate` (`alembic upgrade head`).
```bash
alembic upgrade head
@@ -240,105 +147,6 @@ pytest -q
Тесты работают на SQLite in-memory, без внешних зависимостей.
## `pyproject.toml` + `uv.lock`
Проект переведён на современную схему зависимостей:
- `pyproject.toml` — декларация зависимостей и метаданных проекта
- `uv.lock` — зафиксированные версии для воспроизводимых установок
Локально можно использовать:
```bash
uv sync
uv run pytest -q
```
## Runtime-фильтры
Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
### Секция `sync`
Поддерживаются поля:
- `name`
- `ids_initial_size`
- `ids_next_size`
- `ids_max_pages`
- `condition_check_enabled`
- `lane`
- `only_new`
- `limit`
### Секция `filters`
Поддерживаются поля:
- `brands`
- `models`
- `years`
- `body_types`
- `colors`
- `drives`
- `gearboxes`
- `locations`
- `exclude_brands`
- `exclude_models`
- `exclude_years`
- `exclude_body_types`
- `exclude_colors`
- `exclude_drives`
- `exclude_gearboxes`
- `exclude_locations`
- `price.min`, `price.max`
- `mileage.min`, `mileage.max`
- `flags.damaged_only`, `flags.run_and_drive`
Пример:
```json
{
"sync": {
"name": null,
"ids_initial_size": null,
"ids_next_size": null,
"ids_max_pages": null,
"condition_check_enabled": false,
"lane": "iaai_cars",
"only_new": true,
"limit": 50
},
"filters": {
"price": {
"min": null,
"max": null
},
"mileage": {
"min": null,
"max": null
},
"flags": {
"damaged_only": null,
"run_and_drive": null
},
"brands": ["Toyota", "Honda"],
"models": [],
"years": [2021, 2022],
"body_types": ["SUV"],
"colors": [],
"drives": [],
"gearboxes": [],
"locations": [],
"exclude_brands": [],
"exclude_models": [],
"exclude_years": [],
"exclude_body_types": []
}
}
```
Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.

2
alembic.ini
View File

@@ -3,7 +3,7 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
sqlalchemy.url = postgresql+psycopg2://openlane:openlane@localhost:5432/openlane_scraper
[loggers]
keys = root,sqlalchemy,alembic

4
alembic/env.py
View File

@@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool
# Добавляем корень проекта в sys.path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from iaai_scraper.core.config import Settings
from iaai_scraper.storage.models import Base
from openlane_scraper.core.config import Settings
from openlane_scraper.storage.models import Base
config = context.config

2
alembic/versions/001_initial.py
View File

@@ -11,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Таблица cars — аукционные машины с IAAI
# Таблица cars
op.create_table(
"cars",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),

8
alembic/versions/003_add_composite_indexes.py
View File

@@ -10,12 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Partial index для активных машин IAAI (is_sold = FALSE)
# Partial index для активных машин OpenLane (is_sold = FALSE)
# Ускоряет поиск при mark_sold операциях
op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
"CREATE INDEX IF NOT EXISTS ix_cars_active_openlane "
"ON cars (origin_url) "
"WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
"WHERE is_sold = FALSE AND origin_id LIKE 'openlane:%'"
)
# Composite index для проверки дубликатов изображений
@@ -35,4 +35,4 @@ def upgrade() -> None:
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
op.drop_index("ix_images_car_id_fullres", table_name="images")
op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")
op.execute("DROP INDEX IF EXISTS ix_cars_active_openlane")

76
docker-compose.yml
View File

@@ -1,27 +1,24 @@
x-app-env: &app-env
# NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
# PostgreSQL — единственная БД в Docker
OPENLANE_DATABASE_URL: postgresql+psycopg2://openlane:openlane@postgres:5432/openlane_scraper
OPENLANE_REDIS_URL: ${OPENLANE_REDIS_URL:-redis://redis:6379/0}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
OPENLANE_DATABASE_POOL_RECYCLE_SECONDS: ${OPENLANE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
# 0 => без лимита (в коде интерпретируется как None).
# После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
IAAI_BROWSER_ENGINE: chromium
OPENLANE_MAX_CARS_LIMIT: ${OPENLANE_MAX_CARS_LIMIT:-20}
OPENLANE_THROTTLE_MIN_SECONDS: ${OPENLANE_THROTTLE_MIN_SECONDS:-2.0}
OPENLANE_THROTTLE_MAX_SECONDS: ${OPENLANE_THROTTLE_MAX_SECONDS:-5.0}
OPENLANE_CONCURRENCY: ${OPENLANE_CONCURRENCY:-1}
OPENLANE_MAX_PAGES: ${OPENLANE_MAX_PAGES:-512}
OPENLANE_BROWSER_ENGINE: chromium
OPENLANE_HEADLESS: "true"
OPENLANE_STORAGE_STATE_FILE: /data/openlane/openlane_storage_state.json
OPENLANE_RUNTIME_CONFIG_FILE: /app/runtime_config.json
TZ: ${TZ:-UTC}
x-env-file: &env-file
@@ -40,23 +37,24 @@ x-worker-service: &worker-service
volumes:
- ./runtime_config.json:/app/runtime_config.json:ro
- tokens_data:/data
- openlane_data:/data/openlane
services:
# PostgreSQL
postgres:
image: postgres:16-alpine
container_name: iaai-postgres
container_name: openlane-postgres
restart: unless-stopped
environment:
POSTGRES_USER: iaai
POSTGRES_PASSWORD: iaai
POSTGRES_DB: iaai_scraper
POSTGRES_USER: openlane
POSTGRES_PASSWORD: openlane
POSTGRES_DB: openlane_scraper
ports:
- "127.0.0.1:5432:5432"
- "127.0.0.1:${OPENLANE_POSTGRES_PORT:-55432}:5432"
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
test: ["CMD-SHELL", "pg_isready -U openlane -d openlane_scraper"]
interval: 5s
timeout: 3s
retries: 5
@@ -69,10 +67,10 @@ services:
# Redis (Celery broker)
redis:
image: redis:7-alpine
container_name: iaai-redis
container_name: openlane-redis
restart: unless-stopped
ports:
- "127.0.0.1:6379:6379"
- "127.0.0.1:${OPENLANE_REDIS_PORT:-56379}:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
@@ -93,7 +91,7 @@ services:
# DB migrations
migrate:
<<: *app-service
container_name: iaai-migrate
container_name: openlane-migrate
restart: "no"
depends_on:
postgres:
@@ -103,17 +101,17 @@ services:
# FastAPI
api:
<<: *app-service
container_name: iaai-api
container_name: openlane-api
restart: unless-stopped
ports:
- "127.0.0.1:8000:8000"
- "127.0.0.1:${OPENLANE_API_PORT:-58000}:8000"
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
command: >
uvicorn iaai_scraper.api.app:app
uvicorn openlane_scraper.api.app:app
--host 0.0.0.0 --port 8000 --workers 1
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -131,11 +129,11 @@ services:
# Celery Worker
worker:
<<: *worker-service
container_name: iaai-worker
container_name: openlane-worker
restart: unless-stopped
init: true
mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
mem_limit: ${OPENLANE_WORKER_MEM_LIMIT:-2g}
memswap_limit: ${OPENLANE_WORKER_MEM_LIMIT:-2g}
depends_on:
migrate:
condition: service_completed_successfully
@@ -143,7 +141,7 @@ services:
condition: service_healthy
stop_grace_period: 60s
command: >
celery -A iaai_scraper.worker.celery_app worker
celery -A openlane_scraper.worker.celery_app worker
--loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
--pidfile=/tmp/celery-worker.pid
-Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
@@ -162,7 +160,7 @@ services:
# Celery Beat (периодический планировщик)
beat:
<<: *worker-service
container_name: iaai-beat
container_name: openlane-beat
restart: unless-stopped
init: true
depends_on:
@@ -171,7 +169,7 @@ services:
redis:
condition: service_healthy
command: >
celery -A iaai_scraper.worker.celery_app beat
celery -A openlane_scraper.worker.celery_app beat
--loglevel=info
--pidfile=/tmp/celery-beat.pid
--schedule=/tmp/celerybeat-schedule
@@ -187,7 +185,17 @@ services:
max-size: "10m"
max-file: "5"
openlane-auth:
<<: *worker-service
container_name: openlane-auth
profiles: ["auth"]
stdin_open: true
tty: true
command: >
python scripts/explore_site.py
volumes:
pgdata:
redisdata:
tokens_data:
openlane_data:

56
entrypoint.sh
View File

@@ -1,60 +1,4 @@
#!/bin/bash
set -euo pipefail
is_worker_command() {
local joined="$*"
case "$joined" in
*"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
return 0
;;
esac
return 1
}
is_scrape_cli_command() {
local joined="$*"
case "$joined" in
*"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
return 0
;;
esac
return 1
}
needs_browser_runtime() {
is_worker_command "$@" || is_scrape_cli_command "$@"
}
start_proxy_bridge_if_needed() {
if ! needs_browser_runtime "$@"; then
return 0
fi
if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
return 0
fi
echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
fi
echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
python -m iaai_scraper.proxy_bridge &
BRIDGE_PID=$!
sleep 1
if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
exit 1
fi
echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
}
start_proxy_bridge_if_needed "$@"
exec "$@"

6
iaai_scraper/browser/__init__.py
View File

@@ -1,6 +0,0 @@
from .factory import BrowserFactory
from .listing import ListingCollector
from .network import NetworkCapture
from .pace import HumanPacer
__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

676
iaai_scraper/browser/listing.py
View File

@@ -1,676 +0,0 @@
import logging
import re
import time
from dataclasses import asdict, dataclass, field
from typing import Any
from urllib.parse import urljoin
from playwright.sync_api import Page
from .pace import HumanPacer
from ..core.config import Settings
from ..core.utils import first_non_empty
logger = logging.getLogger("iaai_scraper.listing")
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
"button:has-text('Accept All')",
"button:has-text('Accept all')",
"button:has-text('I Agree')",
"button:has-text('Agree')",
"button:has-text('Only necessary')",
"button:has-text('Только необходимые')",
"button:has-text('Принять все')",
"[id*='accept']",
"[class*='accept']",
)
@dataclass(slots=True)
class ListingVehicleLink:
href: str
title: str = ""
lot_number: str | None = None
@dataclass(slots=True)
class ListingPageResult:
source_url: str
page_number: int
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
pagination_available: bool = False
next_page_detected: bool = False
class ListingCollector:
_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
"a[aria-label*='Next']",
"button[aria-label*='Next']",
"a[aria-label*='next']",
"button[aria-label*='next']",
"a[title*='Next']",
"button[title*='Next']",
"a[title*='next']",
"button[title*='next']",
"a[rel='next']",
"link[rel='next']",
"a.pagination-next",
"button.pagination-next",
"a.next",
"button.next",
"a:has-text('Next')",
"button:has-text('Next')",
"a:has-text('NEXT')",
"button:has-text('NEXT')",
"a:has-text('›')",
"button:has-text('›')",
"a:has-text('»')",
"button:has-text('»')",
"a:has(img[src*='icon-arrow-right'])",
"button:has(img[src*='icon-arrow-right'])",
"a:has(img[src*='arrow-right'])",
"button:has(img[src*='arrow-right'])",
)
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
self.settings = settings
self.pacer = pacer
@staticmethod
def _get_current_page_number(page: Page) -> int | None:
try:
value = page.evaluate(
"""
() => {
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
const current = controls.find((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
});
if (current) {
return parseInt((current.textContent || '').trim(), 10);
}
const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
return match ? parseInt(match[1], 10) : null;
}
"""
)
return int(value) if value is not None else None
except Exception:
return None
@staticmethod
def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=12000,
)
return True
except Exception:
pass
if expected_page_number is not None:
current_page = ListingCollector._get_current_page_number(page)
if current_page == expected_page_number:
return True
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
current_page = ListingCollector._get_current_page_number(page)
if expected_page_number is not None and current_page == expected_page_number:
return True
return not old_first_href
def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
url = url_override or self.settings.listing.cars_url
logger.info("Opening cars listing page: %s", url)
last_err = None
for attempt in range(3):
try:
page.goto(url, wait_until="commit", timeout=60_000)
last_err = None
break
except Exception as e:
last_err = e
logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
# Небольшой backoff при ошибках открытия листинга.
time.sleep(5 * (attempt + 1))
if last_err:
logger.warning("All goto attempts failed, trying JS navigation")
try:
page.evaluate(f"window.location.href = '{url}'")
except Exception:
pass
# Быстрая проверка готовности страницы.
try:
page.wait_for_load_state("domcontentloaded", timeout=12_000)
except Exception:
pass
self._accept_cookie_banner(page)
self._wait_for_listing_content(page)
logger.info("Listing page URL: %s", page.url)
self.pacer.after_listing_open()
def _accept_cookie_banner(self, page: Page) -> None:
for selector in COOKIE_ACCEPT_SELECTORS:
locator = page.locator(selector).first
try:
if locator.count() == 0:
continue
if not locator.is_visible(timeout=500):
continue
locator.click(timeout=2_000)
logger.info("Accepted cookie banner using selector: %s", selector)
try:
page.wait_for_load_state("domcontentloaded", timeout=3_000)
except Exception:
pass
return
except Exception:
continue
def _wait_for_listing_content(self, page: Page) -> None:
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
return
except Exception:
pass
# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
try:
page.wait_for_function(
"""() => {
const html = document.documentElement?.innerHTML || '';
const text = document.body?.innerText || '';
return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
}""",
timeout=12_000,
)
except Exception:
# Короткая пауза вместо длинного sleep.
time.sleep(1.0)
def apply_filters(
self,
page: Page,
make: str | None = None,
model: str | None = None,
year_min: int | None = None,
year_max: int | None = None,
) -> dict[str, str | int | None]:
applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
applied["make"] = make
self.pacer.after_filter_action()
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
applied["model"] = model
self.pacer.after_filter_action()
if year_min is not None or year_max is not None:
if self._apply_year_range(page, year_min, year_max):
applied["year_min"] = year_min
applied["year_max"] = year_max
self.pacer.after_filter_action()
return applied
def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
"""Заполняет поля фильтра Year и нажимает Apply Year."""
if year_min is None and year_max is None:
return False
try:
success = page.evaluate(
"""([yearMin, yearMax]) => {
const inputs = Array.from(document.querySelectorAll('input'));
const yearInputs = inputs.filter(inp => {
const v = parseInt(inp.value, 10);
return !isNaN(v) && v >= 1900 && v <= 2100;
});
if (yearInputs.length < 2) return false;
yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
const setVal = (el, val) => {
const setter = Object.getOwnPropertyDescriptor(
HTMLInputElement.prototype, 'value'
).set;
setter.call(el, String(val));
el.dispatchEvent(new Event('input', {bubbles: true}));
el.dispatchEvent(new Event('change', {bubbles: true}));
};
if (yearMin !== null) setVal(yearInputs[0], yearMin);
if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
const container = yearInputs[0].closest(
'[class*="filter"], [class*="year"], section, fieldset'
) || yearInputs[0].parentElement.parentElement;
if (container) {
const btn = Array.from(container.querySelectorAll(
'button, a, [role="button"], span[class*="apply"]'
)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
if (btn) { btn.click(); return true; }
}
yearInputs[yearInputs.length - 1].dispatchEvent(
new KeyboardEvent('keydown', {
key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
})
);
return true;
}""",
[year_min, year_max],
)
if success:
try:
page.wait_for_load_state("domcontentloaded", timeout=15_000)
except Exception:
pass
self._wait_for_listing_content(page)
logger.info("Applied year range filter: %s — %s", year_min, year_max)
return True
except Exception as exc:
logger.warning("Failed to apply year range filter: %s", exc)
return False
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
# Считываем ссылки одним проходом по DOM.
self._accept_cookie_banner(page)
self._wait_for_listing_content(page)
try:
raw_items = page.eval_on_selector_all(
"a[href], [data-href], [href]",
"""
(nodes) => nodes.map((node) => ({
href:
node.getAttribute('href') ||
node.getAttribute('data-href') ||
node.getAttribute('data-url') ||
'',
title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
text: (node.textContent || '').trim(),
}))
""",
)
except Exception as exc:
logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
raw_items = []
total = min(len(raw_items), self.settings.listing.page_link_limit)
links: list[ListingVehicleLink] = []
seen: set[str] = set()
for idx in range(total):
item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
href = str(item.get("href") or "")
match = VEHICLE_HREF_RE.search(href)
if not match:
continue
lot_number = match.group(1)
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
seen.add(absolute)
title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
if not links:
try:
page.wait_for_timeout(1_500)
except Exception:
pass
try:
html = page.content()
except Exception as exc:
logger.debug("page.content() failed on page %d: %s", page_number, exc)
html = ""
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
if absolute in seen:
continue
seen.add(absolute)
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
if links:
logger.info(
"Page %d: recovered %d vehicle links from HTML fallback",
page_number,
len(links),
)
next_page_detected = self._has_next_page(page)
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
if not html:
return []
# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
normalized = html.replace("\\/", "/")
found: list[tuple[str, str]] = []
seen: set[str] = set()
for match in VEHICLE_HREF_RE.finditer(normalized):
lot_number = match.group(1)
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
seen.add(absolute)
found.append((absolute, lot_number))
if len(found) >= self.settings.listing.page_link_limit:
break
return found
def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
# Запоминаем первую ссылку текущей страницы для определения смены контента.
old_first_href = ""
try:
first_link = page.locator(VEHICLE_LINK_SELECTOR).first
if first_link.count() > 0:
old_first_href = first_link.get_attribute("href") or ""
except Exception:
pass
for selector in self._NEXT_PAGE_SELECTORS:
locator = page.locator(selector).first
if locator.count() == 0:
continue
try:
disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
classes = (locator.get_attribute("class", timeout=1500) or "").lower()
except Exception:
continue
if disabled or aria_disabled == "true" or "disabled" in classes:
continue
try:
self.pacer.move_mouse_to(page, locator)
locator.click(timeout=8000)
except Exception:
continue
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
# + стрелка с иконкой, без явного текста Next.
try:
clicked = bool(page.evaluate(
"""
() => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const disabled = (el) => {
if (!el) return true;
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
};
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
.filter((el) => visible(el) && !disabled(el));
const current = controls.find((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
});
if (current) {
const currentPage = parseInt((current.textContent || '').trim(), 10);
const nextNumber = controls.find((el) => {
const text = (el.textContent || '').trim();
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
});
if (nextNumber) {
nextNumber.click();
return true;
}
}
const iconNext = controls.find((el) => {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
});
if (iconNext) {
iconNext.click();
return true;
}
return false;
}
"""
))
if clicked:
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("Numeric/icon pagination fallback failed: %s", exc)
# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
try:
clicked = bool(page.evaluate(
"""
() => {
const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
for (const el of candidates) {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
if (!disabled && visible && looksNext) {
el.click();
return true;
}
}
return false;
}
"""
))
if clicked:
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("JS next-page fallback failed: %s", exc)
logger.warning("Could not navigate to next page from %s", page.url)
return False
def collect_listing_links(
self,
page: Page,
*,
make: str | None = None,
model: str | None = None,
known_origin_ids: set[str] | None = None,
max_duration_seconds: float | None = None,
) -> dict[str, Any]:
self.open_cars_listing(page)
applied_filters = self.apply_filters(page, make=make, model=model)
started_at = time.perf_counter()
truncated_by_time_budget = False
pages: list[dict[str, object]] = []
all_links: list[str] = []
early_stopped = False
threshold = self.settings.listing.early_stop_threshold
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
if max_duration_seconds is not None and max_duration_seconds > 0:
elapsed = time.perf_counter() - started_at
if elapsed >= max_duration_seconds:
truncated_by_time_budget = True
logger.warning(
"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
page_number,
elapsed,
max_duration_seconds,
)
break
page_result = self.collect_current_page(page, page_number=page_number)
pages.append({
"page_number": page_result.page_number,
"source_url": page_result.source_url,
"links_found": len(page_result.vehicle_links),
"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
"next_page_detected": page_result.next_page_detected,
})
for item in page_result.vehicle_links:
if item.href not in all_links:
all_links.append(item.href)
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
break
# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
if (
known_origin_ids is not None
and threshold > 0.0
and page_result.vehicle_links
):
page_known = sum(
1 for item in page_result.vehicle_links
if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
)
page_new = len(page_result.vehicle_links) - page_known
ratio = page_known / len(page_result.vehicle_links)
# Останавливаемся только если нет новых И порог превышен
if ratio >= threshold and page_new == 0:
logger.info(
"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
page_number, ratio * 100, page_known,
len(page_result.vehicle_links), threshold * 100,
)
early_stopped = True
break
elif page_new > 0 and ratio >= threshold:
logger.info(
"Page %d: %.0f%% known but %d new found — продолжаем",
page_number, ratio * 100, page_new,
)
if (
len(all_links) >= self.settings.listing.max_vehicles_per_run
or self.settings.listing.collect_current_page_only
or not self.settings.listing.include_pagination
or not page_result.next_page_detected
):
break
if not self.go_to_next_page(page):
break
return {
"listing_url": self.settings.listing.cars_url,
"applied_filters": applied_filters,
"pages_collected": len(pages),
"vehicles_collected": len(all_links),
"vehicle_urls": all_links,
"early_stopped": early_stopped,
"truncated_by_time_budget": truncated_by_time_budget,
"pages": pages,
"strategy": {
"sequential": True,
"collect_current_page_only": self.settings.listing.collect_current_page_only,
"include_pagination": self.settings.listing.include_pagination,
"max_pages_per_run": self.settings.listing.max_pages_per_run,
"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
"early_stop_threshold": threshold,
},
}
@staticmethod
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
for selector in selectors:
locator = page.locator(selector).first
if locator.count() == 0:
continue
try:
locator.click()
locator.fill(value)
page.keyboard.press("Enter")
try:
page.wait_for_load_state("domcontentloaded", timeout=15000)
except Exception:
pass
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
return True
except Exception:
continue
return False
@staticmethod
def _has_next_page(page: Page) -> bool:
for selector in ListingCollector._NEXT_PAGE_SELECTORS:
if page.locator(selector).count() > 0:
return True
try:
return bool(page.evaluate(
"""
() => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
return !disabled && visible(el);
});
const hasExplicitNext = controls.some((el) => {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
});
if (hasExplicitNext) {
return true;
}
const current = controls.find((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
});
if (!current) {
return false;
}
const currentPage = parseInt((current.textContent || '').trim(), 10);
return controls.some((el) => {
const text = (el.textContent || '').trim();
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
});
}
"""
))
except Exception:
return False
return False

130
iaai_scraper/browser/network.py
View File

@@ -1,130 +0,0 @@
import json
import logging
from dataclasses import dataclass, field
from typing import Any
from urllib.parse import urlparse
from playwright.sync_api import Page, Request, Response
from ..core.config import Settings
logger = logging.getLogger("iaai_scraper.network")
@dataclass
class NetworkCapture:
# Перехватчик сетевых запросов.
settings: Settings
requests: list[dict[str, Any]] = field(default_factory=list)
json_responses: list[dict[str, Any]] = field(default_factory=list)
_seen_req: set[str] = field(default_factory=set)
_seen_resp: set[str] = field(default_factory=set)
_origin: str | None = None
_page: Any = field(default=None)
def attach(self, page: Page, origin_url: str | None = None) -> None:
# Снимаем старые подписки перед повторным attach.
try:
page.remove_listener("request", self._on_request)
page.remove_listener("response", self._on_response)
except Exception:
pass
# Подписка на сетевые события
try:
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
except Exception:
self._origin = None
self._page = page
page.on("request", self._on_request)
page.on("response", self._on_response)
def _is_same_origin(self, url: str) -> bool:
# Фильтр по домену
if not self.settings.capture.capture_same_origin_only or not self._origin:
return True
netloc = urlparse(url).netloc.lower()
return netloc == self._origin or netloc.endswith(".iaai.com")
def _on_request(self, request: Request) -> None:
# Берём только xhr/fetch
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(request.url):
return
if len(self.requests) >= self.settings.capture.max_requests:
return
key = f"{request.method}:{request.url}:{request.post_data or ''}"
# Убираем дубли
if key in self._seen_req:
return
self._seen_req.add(key)
self.requests.append({
"url": request.url, "method": request.method,
"resource_type": request.resource_type, "post_data": request.post_data,
})
def _on_response(self, response: Response) -> None:
# Сохраняем JSON
request = response.request
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(response.url):
return
if len(self.json_responses) >= self.settings.capture.max_json_responses:
return
if not self._is_json(response):
return
key = f"{request.method}:{response.url}:{response.status}"
if key in self._seen_resp:
return
self._seen_resp.add(key)
try:
payload = response.json()
except Exception:
try:
payload = json.loads(response.text())
except Exception:
return
self.json_responses.append({
"url": response.url, "status": response.status,
"request_method": request.method, "post_data": request.post_data,
"resource_type": request.resource_type, "payload": payload,
"category": self._categorize(response.url),
})
@staticmethod
def _is_json(response: Response) -> bool:
ct = (response.headers.get("content-type") or "").lower()
if "application/json" in ct or "+json" in ct:
return True
url = response.url.lower()
return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
@staticmethod
def _categorize(url: str) -> str:
# Простая категория URL
low = url.lower()
mapping = {
"images": ["image", "media", "photos", "gallery"],
"bids": ["bid", "offer", "buy-now", "buynow"],
"auction": ["auction", "sale", "lane", "branch"],
"vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
"documents": ["title", "document", "report"],
}
for cat, markers in mapping.items():
if any(m in low for m in markers):
return cat
return "other"
def export(self) -> dict[str, Any]:
responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
return {
"requests": self.requests,
"json_responses": responses,
"capture_limits": {
"same_origin_only": self.settings.capture.capture_same_origin_only,
"max_requests": self.settings.capture.max_requests,
"max_json_responses": self.settings.capture.max_json_responses,
},
}

46
iaai_scraper/browser/pace.py
View File

@@ -1,46 +0,0 @@
import random
import time
from playwright.sync_api import Locator, Page
from ..core.config import Settings
class HumanPacer:
# Случайные паузы между действиями.
def __init__(self, settings: Settings) -> None:
self.settings = settings
def pause(self, min_s: float, max_s: float) -> None:
if self.settings.pace.enabled:
time.sleep(random.uniform(min_s, max_s))
def after_listing_open(self) -> None:
self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
def after_filter_action(self) -> None:
self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
def before_vehicle_open(self) -> None:
self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
def after_vehicle_open(self) -> None:
self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
def between_vehicles(self) -> None:
self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
def after_page_change(self) -> None:
self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
def move_mouse_to(self, page: Page, locator: Locator) -> None:
try:
box = locator.bounding_box()
except Exception:
box = None
if not box:
return
x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
page.mouse.move(x, y, steps=random.randint(8, 18))

82
iaai_scraper/cli.py
View File

@@ -1,82 +0,0 @@
import argparse
from pathlib import Path
from .core.config import Settings
from .core.utils import save_to_json
from .scraper import IAAIScraper
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="IAAI scraper CLI")
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
subparsers = parser.add_subparsers(dest="command", required=True)
default_output_dir = Path("artifacts/json")
subparsers.add_parser("init-db", help="Create DB tables")
listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
listing_parser.add_argument("--make", default=None)
listing_parser.add_argument("--model", default=None)
listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
scrape_parser.add_argument("vehicle_url")
scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
sync_vehicle_parser.add_argument("vehicle_url")
sync_vehicle_parser.add_argument("--lane", default="iaai")
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
sync_listing_parser.add_argument("--make", default=None)
sync_listing_parser.add_argument("--model", default=None)
sync_listing_parser.add_argument("--lane", default="iaai_cars")
sync_listing_parser.add_argument("--limit", type=int, default=None)
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
return parser
def main() -> None:
parser = build_parser()
args = parser.parse_args()
runtime_settings: Settings | None = None
if args.headless is not None or args.debug:
runtime_settings = Settings()
if args.headless is not None:
runtime_settings.headless = args.headless == "true"
if args.debug:
runtime_settings.log_level = "DEBUG"
with IAAIScraper(runtime_settings) as scraper:
if args.command == "init-db":
data = scraper.init_db()
print(f"DB initialized: {data}")
return
elif args.command == "collect-listing":
data = scraper.collect_listing(make=args.make, model=args.model)
elif args.command == "scrape-vehicle":
data = scraper.scrape_vehicle_detail(args.vehicle_url)
elif args.command == "sync-vehicle":
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
else:
only_new = None if args.only_new is None else args.only_new == "true"
data = scraper.sync_listing(
make=args.make,
model=args.model,
lane=args.lane,
limit=args.limit,
only_new=only_new,
)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
if __name__ == "__main__":
main()

295
iaai_scraper/core/config.py
View File

@@ -1,295 +0,0 @@
import json
import os
from dataclasses import dataclass, field
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
TRUE_VALUES = {"1", "true", "yes", "on"}
# Хелперы для чтения env-переменных с приведением типов
def _env_str(name: str, default: str) -> str:
value = os.getenv(name)
return value if value is not None else default
def _env_optional_str(name: str) -> str | None:
value = os.getenv(name)
if value is None:
return None
value = value.strip()
return value or None
def _env_path_str(name: str) -> str | None:
value = _env_optional_str(name)
if value is None:
return None
return str(Path(value).expanduser())
def _env_bool(name: str, default: bool) -> bool:
fallback = "true" if default else "false"
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
def _env_int(name: str, default: int) -> int:
return int(_env_str(name, str(default)).strip())
def _env_float(name: str, default: float) -> float:
return float(_env_str(name, str(default)).strip())
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
@dataclass(slots=True)
class FingerprintConfig:
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/135.0.0.0 Safari/537.36"
)
viewport_presets: tuple[dict[str, int], ...] = (
{"width": 1920, "height": 1080},
{"width": 1600, "height": 900},
{"width": 1536, "height": 864},
{"width": 1440, "height": 900},
{"width": 1366, "height": 768},
)
timezone_candidates: tuple[str, ...] = (
"America/New_York",
"America/Chicago",
"America/Los_Angeles",
)
locale: str = "en-US"
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
@dataclass(slots=True)
class CaptureConfig:
capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
# Конфиг пауз между действиями (имитация человека)
@dataclass(slots=True)
class HumanPaceConfig:
enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
# Конфиг сбора листинга (URL, лимиты страниц и машин)
@dataclass(slots=True)
class ListingConfig:
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
# Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
# JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
# Пустая строка = без сегментации (backward compatible).
listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
# Список брендов IAAI для автоматической сегментации.
# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
IAAI_DEFAULT_MAKES: tuple[str, ...] = (
"TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
"KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
"VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
"RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
"MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
"MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
"SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
"OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
"RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
"BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
"MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
)
# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
IAAI_PAGINATION_CEILING = 22_600
# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
_LARGE_MAKES: frozenset[str] = frozenset({
"TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
"KIA", "DODGE", "JEEP",
})
_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
(1900, 2012),
(2013, 2019),
(2020, 2027),
)
def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
"""Парсит IAAI_LISTING_SEGMENTS в список сегментов.
Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
Крупные бренды автоматически разбиваются по диапазонам годов.
"""
raw = raw.strip()
if not raw:
return []
if raw.lower() == "auto":
segments: list[dict[str, str | int | None]] = []
for m in IAAI_DEFAULT_MAKES:
if m in _LARGE_MAKES:
for yr_min, yr_max in _YEAR_SPLITS:
segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
else:
segments.append({"make": m, "year_min": None, "year_max": None})
return segments
try:
data = json.loads(raw)
except (json.JSONDecodeError, ValueError):
return []
if not isinstance(data, list):
return []
segments = []
for item in data:
if isinstance(item, str):
segments.append({"make": item.upper(), "year_min": None, "year_max": None})
elif isinstance(item, dict):
segments.append({
"make": str(item.get("make") or "").upper() or None,
"year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
"year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
})
return segments
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
@dataclass(slots=True)
class DatabaseConfig:
url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
# --- Конфиг Redis (URL для Celery broker) ---
@dataclass(slots=True)
class RedisConfig:
url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@dataclass(slots=True)
class CeleryConfig:
broker_url: str = _env_str("CELERY_BROKER_URL", "")
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
# --- Конфиг прокси (server, username, password) ---
@dataclass(slots=True)
class ProxyConfig:
server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
@property
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
if self.username:
result["username"] = self.username
if self.password:
result["password"] = self.password
return result
# Главный объект настроек: собирает все блоки конфигурации
@dataclass(slots=True)
class Settings:
home_url: str = "https://www.iaai.com/"
default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
headless: bool = _env_bool("IAAI_HEADLESS", True)
browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
capture: CaptureConfig = field(default_factory=CaptureConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
@property
def parallel_tabs(self) -> int:
return self.celery.parallel_tabs
@property
def block_resources(self) -> bool:
return self.celery.block_resources
# Глобальный синглтон — используется по умолчанию во всех модулях.
settings = Settings()

14
iaai_scraper/core/exceptions.py
View File

@@ -1,14 +0,0 @@
class ScraperError(Exception):
"""Базовое исключение скрапера."""
class AntiBotDetectedError(ScraperError):
"""Вызывается, когда сайт блокирует автоматизацию."""
class SiteStructureChangedError(ScraperError):
"""Вызывается, когда структура страницы изменилась и данных не хватает."""
class ListingResumeError(ScraperError):
"""Вызывается, когда resume по checkpoint больше недостижим."""

53
iaai_scraper/core/retry.py
View File

@@ -1,53 +0,0 @@
import logging
import random
import time
from collections.abc import Callable
from functools import wraps
from typing import Any
from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
from .exceptions import AntiBotDetectedError
logger = logging.getLogger("iaai_scraper.retry")
# Типы исключений, при которых retry имеет смысл.
RETRYABLE_EXCEPTIONS = (
PlaywrightTimeoutError,
Error,
ConnectionError,
OSError,
TimeoutError,
AntiBotDetectedError,
)
def retryable(
max_attempts: int,
delay_seconds: float = 2.5,
backoff_multiplier: float = 2.0,
jitter_seconds: float = 0.0,
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
@wraps(func)
def wrapper(*args: Any, **kwargs: Any) -> Any:
last_error: Exception | None = None
for attempt in range(1, max_attempts + 1):
try:
return func(*args, **kwargs)
except RETRYABLE_EXCEPTIONS as exc:
last_error = exc
logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
if attempt < max_attempts:
sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
if jitter_seconds > 0:
sleep_for += random.uniform(0, jitter_seconds)
logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
time.sleep(sleep_for)
if last_error is not None:
raise last_error
raise RuntimeError("Retry wrapper failed without a captured exception")
return wrapper
return decorator

301
iaai_scraper/core/runtime_config.py
View File

@@ -1,301 +0,0 @@
import json
import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
logger = logging.getLogger("iaai_scraper.runtime_config")
def _normalize_text(value: str) -> str:
return value.strip().casefold()
def _text_tuple(values: Any) -> tuple[str, ...]:
return tuple(
str(item).strip()
for item in (values or [])
if str(item).strip()
)
def _int_tuple(values: Any) -> tuple[int, ...]:
result: list[int] = []
for value in values or []:
try:
result.append(int(value))
except (TypeError, ValueError):
continue
return tuple(result)
def _optional_int(value: Any) -> int | None:
if value in (None, ""):
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _optional_bool(value: Any) -> bool | None:
if value is None:
return None
if isinstance(value, bool):
return value
if isinstance(value, str):
normalized = value.strip().casefold()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return None
@dataclass(slots=True)
class RuntimeSyncConfig:
name: str | None = None
ids_initial_size: int | None = None
ids_next_size: int | None = None
ids_max_pages: int | None = None
condition_check_enabled: bool | None = None
lane: str | None = None
only_new: bool | None = None
limit: int | None = None
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig":
data = data or {}
name = str(data.get("name")).strip() if data.get("name") else None
lane = str(data.get("lane")).strip() if data.get("lane") else None
return cls(
name=name or None,
ids_initial_size=_optional_int(data.get("ids_initial_size")),
ids_next_size=_optional_int(data.get("ids_next_size")),
ids_max_pages=_optional_int(data.get("ids_max_pages")),
condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
lane=lane or None,
only_new=_optional_bool(data.get("only_new")),
limit=_optional_int(data.get("limit")),
)
@dataclass(slots=True)
class RuntimeListingConfig:
make: str | None = None
model: str | None = None
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig":
data = data or {}
make = str(data.get("make")).strip() if data.get("make") else None
model = str(data.get("model")).strip() if data.get("model") else None
return cls(make=make or None, model=model or None)
@dataclass(slots=True)
class RuntimeFieldFilters:
brands: tuple[str, ...] = ()
models: tuple[str, ...] = ()
years: tuple[int, ...] = ()
body_types: tuple[str, ...] = ()
colors: tuple[str, ...] = ()
drives: tuple[str, ...] = ()
gearboxes: tuple[str, ...] = ()
locations: tuple[str, ...] = ()
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters":
data = data or {}
return cls(
brands=_text_tuple(data.get("brands")),
models=_text_tuple(data.get("models")),
years=_int_tuple(data.get("years")),
body_types=_text_tuple(data.get("body_types")),
colors=_text_tuple(data.get("colors")),
drives=_text_tuple(data.get("drives")),
gearboxes=_text_tuple(data.get("gearboxes")),
locations=_text_tuple(data.get("locations")),
)
def is_empty(self) -> bool:
return not any([
self.brands,
self.models,
self.years,
self.body_types,
self.colors,
self.drives,
self.gearboxes,
self.locations,
])
def matches(self, values: dict[str, Any]) -> bool:
return all([
self._match_text(self.brands, values.get("brand")),
self._match_text(self.models, values.get("model")),
self._match_int(self.years, values.get("year")),
self._match_text(self.body_types, values.get("body_type")),
self._match_text(self.colors, values.get("color")),
self._match_text(self.drives, values.get("drive")),
self._match_text(self.gearboxes, values.get("gearbox")),
self._match_text(self.locations, values.get("location")),
])
@staticmethod
def _match_text(allowed: tuple[str, ...], value: Any) -> bool:
if not allowed:
return True
normalized = _normalize_text(str(value or ""))
return normalized in {_normalize_text(item) for item in allowed}
@staticmethod
def _match_int(allowed: tuple[int, ...], value: Any) -> bool:
if not allowed:
return True
parsed = _optional_int(value)
return parsed in set(allowed)
@dataclass(slots=True)
class RuntimeRangeFilter:
min: int | None = None
max: int | None = None
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter":
data = data or {}
return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max")))
def is_empty(self) -> bool:
return self.min is None and self.max is None
def matches(self, value: Any) -> bool:
parsed = _optional_int(value)
if parsed is None:
return self.is_empty()
if self.min is not None and parsed < self.min:
return False
if self.max is not None and parsed > self.max:
return False
return True
@dataclass(slots=True)
class RuntimeFlagFilters:
damaged_only: bool | None = None
run_and_drive: bool | None = None
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters":
data = data or {}
return cls(
damaged_only=_optional_bool(data.get("damaged_only")),
run_and_drive=_optional_bool(data.get("run_and_drive")),
)
def is_empty(self) -> bool:
return self.damaged_only is None and self.run_and_drive is None
def matches(self, values: dict[str, Any]) -> bool:
if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only:
return False
if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive:
return False
return True
@dataclass(slots=True)
class RuntimeFiltersConfig:
include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters)
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig":
data = data or {}
legacy_fields = RuntimeFieldFilters.from_dict(data)
include_payload = data.get("include")
exclude_payload = data.get("exclude")
flat_exclude_payload = {
"brands": data.get("exclude_brands"),
"models": data.get("exclude_models"),
"years": data.get("exclude_years"),
"body_types": data.get("exclude_body_types"),
"colors": data.get("exclude_colors"),
"drives": data.get("exclude_drives"),
"gearboxes": data.get("exclude_gearboxes"),
"locations": data.get("exclude_locations"),
}
include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields
exclude = (
RuntimeFieldFilters.from_dict(exclude_payload)
if exclude_payload is not None
else RuntimeFieldFilters.from_dict(flat_exclude_payload)
)
return cls(
include=include,
exclude=exclude,
price=RuntimeRangeFilter.from_dict(data.get("price")),
mileage=RuntimeRangeFilter.from_dict(data.get("mileage")),
flags=RuntimeFlagFilters.from_dict(data.get("flags")),
)
def is_empty(self) -> bool:
return all([
self.include.is_empty(),
self.exclude.is_empty(),
self.price.is_empty(),
self.mileage.is_empty(),
self.flags.is_empty(),
])
def matches(self, values: dict[str, Any]) -> bool:
if not self.include.matches(values):
return False
if not self._matches_exclude(values):
return False
if not self.price.matches(values.get("price")):
return False
if not self.mileage.matches(values.get("mileage")):
return False
if not self.flags.matches(values):
return False
return True
def _matches_exclude(self, values: dict[str, Any]) -> bool:
if self.exclude.is_empty():
return True
return not self.exclude.matches(values)
@dataclass(slots=True)
class RuntimeConfig:
sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
@classmethod
def from_file(cls, config_path: str | None) -> "RuntimeConfig":
if not config_path:
return cls()
path = Path(config_path)
if not path.exists():
logger.info("Runtime config file not found: %s", path)
return cls()
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception as exc:
logger.warning("Failed to read runtime config %s: %s", path, exc)
return cls()
return cls(
sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
listing=RuntimeListingConfig.from_dict(payload.get("listing")),
filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
)

405
iaai_scraper/parsing/mapper.py
View File

@@ -1,405 +0,0 @@
import hashlib
import re
from datetime import datetime, timezone
from string import ascii_letters, digits
from typing import Any
from urllib.parse import urlparse
from ..core.utils import first_non_empty
from ..storage.enums import (
BODY_TYPE_ENUM_VALUES,
COUNTRY_ENUM_VALUES,
CURRENCY_ENUM_VALUES,
DRIVE_ENUM_VALUES,
GEARBOX_ENUM_VALUES,
ORIGIN_ENUM_VALUES,
SELLING_TYPE_ENUM_VALUES,
STEERING_WHEEL_ENUM_VALUES,
)
from ..storage.schemas import CarRecord, ImageRecord
class CarMapper:
# Преобразование данных IAAI в CarRecord.
BODY_MAP = {
"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
}
DRIVE_MAP = {
"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
}
GEARBOX_MAP = {
"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
}
STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
COUNTRY_MAP = {
"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
}
NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
# Собираем нормализованную DB-модель.
vehicle_summary = vehicle_summary or {}
payload_insights = payload_insights or {}
core = payload_insights.get("vehicle_core", {})
pricing = payload_insights.get("pricing", {})
damage = payload_insights.get("damage", {})
auction = payload_insights.get("auction", {})
images = payload_insights.get("images", {})
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
parser_id = self._generate_parser_id(origin_id)
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
price = self._first_parsed_int(
[
pricing.get("buy_now"),
pricing.get("current_bid"),
vehicle_summary.get("buy_now"),
vehicle_summary.get("current_bid"),
pricing.get("actual_cash_value"),
],
self._to_money_int,
)
mileage = self._parse_odometer(
first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
)
color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
# Пытаемся определить привод из строки двигателя.
if not drive or drive == "NA":
engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
if engine_text:
inferred_drive = self._normalize_drive(engine_text)
if inferred_drive and inferred_drive != "NA":
drive = inferred_drive
gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
is_damaged = self._bool_damage(damage, vehicle_summary)
is_sold = self._bool_sold(auction)
one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
currency = self._normalize_currency(
first_non_empty(
[
pricing.get("currency"),
vehicle_summary.get("currency"),
pricing.get("buy_now"),
pricing.get("current_bid"),
vehicle_summary.get("buy_now"),
vehicle_summary.get("current_bid"),
"USD",
]
)
)
slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
origin = "IAAI"
return CarRecord(
parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
)
@staticmethod
def _as_str(value: Any) -> str:
return "" if value is None else str(value).strip()
@staticmethod
def _to_int(value: Any) -> int | None:
if value is None:
return None
if isinstance(value, bool):
return int(value)
if isinstance(value, (int, float)):
return int(value)
digits = re.sub(r"[^\d]", "", str(value))
return int(digits) if digits else None
@classmethod
def _to_money_int(cls, value: Any) -> int | None:
# Нормализация стоимости из разных форматов.
if value is None:
return None
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
return int(value)
text = str(value).strip()
if not text:
return None
lowered = text.lower()
if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
return None
numbers = re.findall(r"\d[\d\s.,]*", text)
if not numbers:
return None
best: int | None = None
for number in numbers:
clean = number.replace(" ", "")
if "," in clean and "." in clean:
# Поддержка 1,234.56 и 1.234,56.
if clean.rfind(",") > clean.rfind("."):
clean = clean.replace(".", "").replace(",", ".")
else:
clean = clean.replace(",", "")
elif "," in clean:
parts = clean.split(",")
# 123,45 -> 123.45, иначе разделитель тысяч.
if len(parts[-1]) in {1, 2} and len(parts) == 2:
clean = clean.replace(",", ".")
else:
clean = clean.replace(",", "")
elif "." in clean:
parts = clean.split(".")
if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
clean = clean.replace(".", "")
try:
parsed = int(float(clean))
except ValueError:
continue
if parsed > 0 and (best is None or parsed > best):
best = parsed
return best
@staticmethod
def _first_parsed_int(values: list[Any], parser) -> int | None:
for value in values:
parsed = parser(value)
if parsed is not None:
return parsed
return None
@staticmethod
def _to_year(value: Any) -> int | None:
parsed = CarMapper._to_int(value)
if parsed is None:
return None
if 1900 <= parsed <= 2100:
return parsed
return None
@staticmethod
def _parse_odometer(value: Any) -> int:
# Разбор пробега из строк IAAI.
if value is None:
return 0
text = str(value).strip()
if not text:
return 0
lowered = text.lower()
if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
return 0
# Извлекаем число из строкового формата одометра.
numbers = re.findall(r"[\d,]+", text)
for num_str in numbers:
clean = num_str.replace(",", "")
if clean.isdigit() and int(clean) > 0:
return int(clean)
return 0
def _to_engine_cc(self, value: Any) -> int | None:
# Поддержка литров и cc.
text = str(value).lower().strip() if value is not None else ""
if not text:
return None
if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
return int(float(liters_match.group(1)) * 1000)
if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
return int(cubic_match.group(1))
return int(text) if re.match(r"^\d+$", text) else None
def _normalize_currency(self, value: Any) -> str:
text = self._as_str(value)
upper = text.upper() or "USD"
if any(token in text for token in ["€", "EUR"]):
return "EUR"
if any(token in text for token in ["¥", "JPY"]):
return "JPY"
if any(token in text for token in ["₩", "KRW"]):
return "KRW"
if any(token in text for token in ["£", "GBP"]):
return "GBP"
if any(token in text for token in ["₽", "RUB"]):
return "RUB"
if any(token in text for token in ["AED", "د.إ"]):
return "AED"
if any(token in text for token in ["CA$", "CAD"]):
return "CAD"
text = upper
if text in CURRENCY_ENUM_VALUES:
return text
return "USD" if "$" in str(value) else "USD"
def _normalize_drive(self, value: Any) -> str | None:
return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
def _normalize_gearbox(self, value: Any) -> str | None:
return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
def _normalize_steering(self, value: Any) -> str | None:
return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
def _normalize_body_type(self, value: Any) -> str:
return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
def _normalize_country(self, value: Any) -> str:
fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
def _normalize_selling_type(self, value: Any) -> str:
text = self._as_str(value) or "AUCTION"
return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
def _map_value(
self,
value: Any,
mapping: dict[str, str],
allowed_values: tuple[str, ...],
*,
empty_default: str | None,
fallback: str | None,
) -> str | None:
# Общий helper для enum-нормализации.
text = self._as_str(value).lower()
if not text:
return empty_default
if (mapped := mapping.get(text)) and mapped in allowed_values:
return mapped
for marker, mapped in mapping.items():
if marker in text and mapped in allowed_values:
return mapped
return fallback
@staticmethod
def _normalize_color(value: Any) -> str:
text = str(value).strip() if value is not None else "other"
if not text:
return "other"
if "/" in text:
text = text.split("/")[0].strip()
return text.lower() or "other"
@staticmethod
def _boolish(value: Any) -> bool:
return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
text = self._as_str(value).lower()
if text and text not in self.NO_DAMAGE_MARKERS:
return True
return False
def _bool_sold(self, auction: dict[str, Any]) -> bool:
return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
# Для imageKeys берем самый большой размер.
best_by_key: dict[str, str] = {}
key_order: list[str] = []
non_keyed: list[str] = []
for item in urls:
if not isinstance(item, str):
continue
url = item.strip()
if not url:
continue
if m := re.search(r'imageKeys=([^&]+)', url):
img_key = m.group(1)
w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
existing = best_by_key.get(img_key)
if existing is None:
best_by_key[img_key] = url
key_order.append(img_key)
else:
existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
if w > existing_w:
best_by_key[img_key] = url
elif url not in non_keyed:
non_keyed.append(url)
deduped = [best_by_key[k] for k in key_order] + non_keyed
return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
@staticmethod
def _make_fullres_url(url: str) -> str:
if "vis.iaai.com" in url:
return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
return url
@staticmethod
def _make_preview_url(url: str) -> str:
if "vis.iaai.com" in url:
preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
if preview != url:
return preview
return url
# Формирование parser_id.
_PARSER_ID_ALPHABET = ascii_letters + digits
@classmethod
def _generate_parser_id(cls, origin_id: str) -> str:
# Стабильный parser_id по origin_id.
digest = hashlib.sha256(origin_id.encode()).digest()
alphabet = cls._PARSER_ID_ALPHABET
base = len(alphabet)
num = int.from_bytes(digest[:17], "big") # 17 байт = 136 бит, хватает на 22 символа
chars: list[str] = []
for _ in range(22):
num, idx = divmod(num, base)
chars.append(alphabet[idx])
return "car-" + "".join(chars)
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
# Формат: iaai:{lot_number} (аналог copart:94323985).
for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
text = self._as_str(value)
if text:
return f"iaai:{text}"
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
if "~" in tail:
tail = tail.split("~")[0]
raw = tail or self._slugify(vehicle_url)
return f"iaai:{raw}"
@staticmethod
def _slugify(value: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"

408
iaai_scraper/parsing/parser.py
View File

@@ -1,408 +0,0 @@
import html as html_module
import json
import logging
import re
from typing import Any
from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
logger = logging.getLogger("iaai_scraper.parsers")
class VehicleParser:
# Парсер данных страницы автомобиля.
# Регулярные выражения для парсинга и детекции защиты.
_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
SUMMARY_KEY_MAP = {
"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
"year": {"year"},
"make": {"make", "manufacturer", "brand"},
"model": {"model"},
"trim": {"trim", "series"},
"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
"secondary_damage": {"secondarydamage"},
"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
"actual_cash_value": {"actualcashvalue", "acv"},
"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
"keys": {"keys", "keystatus"},
"title": {"titletype", "title", "documenttype"},
"seller": {"seller", "sellername"},
"location": {"location", "branchname", "auctionlocation", "branch"},
"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
"gearbox": {"transmission", "gearbox", "transmissiontype"},
"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
"fuel_type": {"fueltype", "fuel"},
"cylinders": {"cylinders", "cylindercount"},
"color": {"color", "primarycolor", "exteriorcolor"},
}
DOM_LABEL_MAP: dict[str, str] = {
"stock #": "lot_number",
"stock": "lot_number",
"primary damage": "primary_damage",
"secondary damage": "secondary_damage",
"odometer": "odometer",
"odometer (miles)": "odometer",
"mileage": "odometer",
"body style": "body_type",
"body type": "body_type",
"vehicle type": "body_type",
"engine": "engine",
"engine type": "engine",
"transmission": "gearbox",
"drive line type": "drive",
"driveline type": "drive",
"drive line": "drive",
"driveline": "drive",
"drive type": "drive",
"fuel type": "fuel_type",
"fuel": "fuel_type",
"cylinders": "cylinders",
"exterior/interior": "color",
"exterior color": "color",
"color": "color",
"model": "model",
"series": "trim",
"selling branch": "location",
"vehicle location": "vehicle_location",
"auction date and time": "auction_date",
"sale date": "auction_date",
"lane/run #": "lane",
"actual cash value": "actual_cash_value",
"estimated repair cost": "estimated_repair_cost",
"seller": "seller",
"title/sale doc": "title",
"title/sale doc brand": "title_brand",
"start code": "run_and_drive",
"key": "keys",
"keys": "keys",
"manufactured in": "manufactured_in",
"vehicle class": "vehicle_class",
}
def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
result: dict[str, str] = {}
if not dom_text:
return result
lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
known_labels = set(self.DOM_LABEL_MAP.keys())
skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
max_fields = len(set(self.DOM_LABEL_MAP.values()))
for i, line in enumerate(lines):
# Ранний выход, когда уже нашли все поля.
if len(result) >= max_fields:
break
# Сценарий 1: "метка: значение" в одной строке.
colon_pos = line.find(":")
if colon_pos > 0:
label_part = line[:colon_pos].strip().lower()
value_part = line[colon_pos + 1:].strip()
if label_part in known_labels and value_part and value_part.lower() not in skip_values:
field_name = self.DOM_LABEL_MAP[label_part]
if field_name not in result or not result[field_name]:
result[field_name] = value_part
continue
# Сценарий 2: метка и значение на соседних строках.
clean = line.rstrip(":").strip().lower()
clean_alt = clean.rstrip("#").strip()
matched_label = None
if clean in known_labels:
matched_label = clean
elif clean_alt in known_labels:
matched_label = clean_alt
if matched_label and i + 1 < len(lines):
value = lines[i + 1].strip()
if value.rstrip(":").lower().strip() in known_labels:
continue
if value.lower() in skip_values:
continue
field_name = self.DOM_LABEL_MAP[matched_label]
if field_name not in result or not result[field_name]:
result[field_name] = value
return result
def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
result: dict[str, str | None] = {"year": None, "make": None, "model": None}
title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
if title_match:
result["year"] = title_match.group(1)
result["make"] = title_match.group(2)
result["model"] = title_match.group(3)
return result
dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
if dom_match:
result["year"] = dom_match.group(1)
result["make"] = dom_match.group(2)
result["model"] = dom_match.group(3).strip()
return result
def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
page_html = page_html or ""
dom_text = dom_text or ""
network_dump = network_dump or {}
responses = network_dump.get("json_responses", [])
payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
dom_kv = self._parse_dom_key_value_pairs(dom_text)
page_title = ""
title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
if title_match:
page_title = title_match.group(1).strip()
title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
# Один проход по payload для всех полей.
all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
summary: dict[str, Any] = {"source_url": vehicle_url}
for field, values in all_found.items():
if field in dom_kv:
values.append(dom_kv[field])
summary[field] = first_non_empty(values)
summary["year"] = summary.get("year") or title_parsed.get("year")
summary["make"] = summary.get("make") or title_parsed.get("make")
summary["model"] = summary.get("model") or title_parsed.get("model")
summary["trim"] = summary.get("trim") or dom_kv.get("trim")
summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
for dom_field, dom_value in dom_kv.items():
if dom_field not in summary or not summary[dom_field]:
summary[dom_field] = dom_value
prices = self._extract_prices_from_text(dom_text)
if not summary.get("actual_cash_value") and prices:
summary["actual_cash_value"] = prices[0]
if not summary.get("buy_now"):
buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
if buy_now_match:
summary["buy_now"] = buy_now_match.group(1)
elif prices:
summary["buy_now"] = prices[0]
if not summary.get("current_bid") and len(prices) > 1:
summary["current_bid"] = prices[1]
embedded = self._extract_embedded_json(page_html)
for item in embedded:
p = item.get("payload")
if isinstance(p, (dict, list)):
payloads.append(p)
# Дополнительный проход по встроенному JSON.
extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
for field, vals in extra.items():
if not summary.get(field):
v = first_non_empty(vals)
if v:
summary[field] = v
# Передаём image_urls без повторного извлечения.
image_urls = summary.get("image_urls") or []
return {
"vehicle_summary": summary,
"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
"embedded_json": embedded,
"dom_hints": self._dom_hints(dom_text),
"access_notes": self._build_access_notes(summary, responses),
}
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
if image_urls is None:
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
return {
"vehicle_core": {
"lot_number": summary.get("lot_number"), "year": summary.get("year"),
"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
"color": summary.get("color"), "keys": summary.get("keys"),
},
"pricing": {
"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
"currency": self._guess_currency(summary),
},
"bids": self._build_bid_insights(summary, payloads),
"damage": {
"primary": summary.get("primary_damage"),
"secondary": summary.get("secondary_damage"),
"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
},
"auction": {
"auction_date": summary.get("auction_date"),
"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
},
"images": {"count": len(image_urls), "urls": image_urls},
"source_endpoints": self._build_source_endpoints(responses),
}
def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
return {
"amount": summary.get("current_bid"),
"currency": self._guess_currency(summary),
"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
}
def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
for item in responses:
url = item.get("url", "")
category = item.get("category", "other")
if category == "vehicle":
mapping["vehicle"].append(url)
lowered = url.lower()
if any(token in lowered for token in ["bid", "offer"]):
mapping["bids"].append(url)
if any(token in lowered for token in ["damage", "report"]):
mapping["damage"].append(url)
if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
mapping["auction"].append(url)
elif category in mapping:
mapping[category].append(url)
return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
endpoints = [item.get("url", "") for item in responses]
dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
return {
"images_visible": bool(summary.get("image_urls")),
"network_json_count": len(responses),
"possible_captcha": bool(dom_hints.get("has_captcha_text")),
"possible_antibot": bool(dom_hints.get("has_antibot_text")),
"observed_endpoints": endpoints[:20],
}
@staticmethod
def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
lowered = {key.lower() for key in keys}
values: list[Any] = []
for payload in payloads:
values.extend(deep_find_key(payload, lowered))
return values
@staticmethod
def _guess_currency(summary: dict[str, Any]) -> str:
for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
value = str(summary.get(key) or "")
if "$" in value:
return "USD"
if "€" in value:
return "EUR"
if "¥" in value:
return "JPY"
return "USD"
@staticmethod
def _extract_lot_number(text: str) -> str | None:
match = LOT_RE.search(text or "")
return match.group(1) if match else None
@staticmethod
def _extract_prices_from_text(text: str) -> list[str]:
return [match.group(1) for match in PRICE_RE.finditer(text or "")]
@staticmethod
def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
extracted: list[dict[str, Any]] = []
for script_text in scripts:
if "{" not in script_text and "[" not in script_text:
continue
# Пропускаем слишком большие минифицированные блоки.
if len(script_text) > 51_200:
continue
try:
parsed = json.loads(script_text.strip())
except Exception:
continue
extracted.append({"type": "inline_json", "payload": parsed})
return extracted
@staticmethod
def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
vehicle_key = ""
key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
if key_match:
vehicle_key = key_match.group(1)
found: list[str] = []
for payload in payloads:
found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
flat: list[str] = []
seen_flat: set[str] = set()
for item in found:
if isinstance(item, str) and item.startswith("http"):
cleaned = html_module.unescape(item)
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
continue
if cleaned not in seen_flat:
seen_flat.add(cleaned)
flat.append(cleaned)
elif isinstance(item, list):
for child in item:
if isinstance(child, str) and child.startswith("http"):
cleaned = html_module.unescape(child)
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
continue
if cleaned not in seen_flat:
seen_flat.add(cleaned)
flat.append(cleaned)
for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
for match in re.finditer(pattern, html or "", re.IGNORECASE):
url = html_module.unescape(match.group(1).strip())
if not url.startswith("http") or url in seen_flat:
continue
lowered = url.lower()
if vehicle_key and vehicle_key in url:
seen_flat.add(url)
flat.append(url)
elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
if vehicle_key and vehicle_key not in url:
continue
seen_flat.add(url)
flat.append(url)
if vehicle_key:
for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
cleaned = html_module.unescape(url)
if cleaned not in seen_flat and vehicle_key in cleaned:
seen_flat.add(cleaned)
flat.append(cleaned)
filtered: list[str] = []
for url in flat:
lowered = url.lower()
if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
continue
if "vis.iaai.com" in lowered and "/resizer" not in lowered:
continue
filtered.append(url)
return filtered
@staticmethod
def _dom_hints(text: str) -> dict[str, Any]:
lowered = (text or "").lower()
return {
"has_buy_now_text": "buy now" in lowered,
"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
"has_damage_text": "damage" in lowered,
"has_title_text": "title" in lowered,
"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
}

317
iaai_scraper/proxy_bridge.py
View File

@@ -1,317 +0,0 @@
from __future__ import annotations
import logging
import os
import select
import socket
import socketserver
import struct
import threading
from urllib.parse import urlsplit
BUFFER_SIZE = 65536
CRLF = b"\r\n"
DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1")
DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))
SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "")
SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "")
RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60"))
MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64"))
logger = logging.getLogger("proxy_bridge")
class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
allow_reuse_address = True
daemon_threads = True
def __init__(self, server_address, request_handler_class):
super().__init__(server_address, request_handler_class)
self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS)
def process_request_thread(self, request, client_address):
with self._worker_semaphore:
super().process_request_thread(request, client_address)
def _recv_exact(sock: socket.socket, size: int) -> bytes:
data = b""
while len(data) < size:
chunk = sock.recv(size - len(data))
if not chunk:
raise ConnectionError("Unexpected EOF from SOCKS5 server")
data += chunk
return data
def _socks5_connect(host: str, port: int) -> socket.socket:
if not SOCKS5_HOST:
raise RuntimeError("SOCKS5_PROXY_HOST is not configured")
upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30)
upstream.settimeout(30)
methods = [0x00]
if SOCKS5_USER or SOCKS5_PASS:
methods = [0x02]
upstream.sendall(bytes([0x05, len(methods), *methods]))
version, method = _recv_exact(upstream, 2)
if version != 0x05 or method == 0xFF:
upstream.close()
raise ConnectionError("SOCKS5 authentication negotiation failed")
if method == 0x02:
username = SOCKS5_USER.encode("utf-8")
password = SOCKS5_PASS.encode("utf-8")
if len(username) > 255 or len(password) > 255:
upstream.close()
raise ValueError("SOCKS5 username/password too long")
upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password)
auth_version, auth_status = _recv_exact(upstream, 2)
if auth_version != 0x01 or auth_status != 0x00:
upstream.close()
raise ConnectionError("SOCKS5 username/password authentication failed")
try:
socket.inet_aton(host)
addr_type = 0x01
addr_payload = socket.inet_aton(host)
except OSError:
host_bytes = host.encode("idna")
if len(host_bytes) > 255:
upstream.close()
raise ValueError("Target host is too long for SOCKS5 domain format")
addr_type = 0x03
addr_payload = bytes([len(host_bytes)]) + host_bytes
request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port)
upstream.sendall(request)
response_head = _recv_exact(upstream, 4)
version, reply, _reserved, reply_addr_type = response_head
if version != 0x05 or reply != 0x00:
upstream.close()
raise ConnectionError(f"SOCKS5 connect failed with code {reply}")
if reply_addr_type == 0x01:
_recv_exact(upstream, 4)
elif reply_addr_type == 0x03:
domain_len = _recv_exact(upstream, 1)[0]
_recv_exact(upstream, domain_len)
elif reply_addr_type == 0x04:
_recv_exact(upstream, 16)
_recv_exact(upstream, 2)
upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
return upstream
def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None:
sockets = [left, right]
left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
try:
while True:
readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS)
if exceptional:
break
if not readable:
logger.debug("Relay idle timeout reached; closing sockets")
return
for current in readable:
other = right if current is left else left
data = current.recv(BUFFER_SIZE)
if not data:
return
other.sendall(data)
finally:
for sock in sockets:
try:
sock.shutdown(socket.SHUT_RDWR)
except OSError:
pass
try:
sock.close()
except OSError:
pass
class ProxyHandler(socketserver.StreamRequestHandler):
def handle(self) -> None:
try:
request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip()
if not request_line:
return
method, target, version = request_line.split()
headers = self._read_headers()
logger.info("%s %s", method, target)
if method.upper() == "CONNECT":
host, port = self._parse_connect_target(target)
logger.info("CONNECT %s:%s", host, port)
upstream = _socks5_connect(host, port)
self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF)
self.wfile.flush()
_relay_bidirectional(self.connection, upstream)
return
host, port, path = self._parse_forward_target(target, headers)
upstream = _socks5_connect(host, port)
self._send_forward_request(upstream, method, path, version, headers)
body = self._read_request_body(headers)
if body:
upstream.sendall(body)
_relay_bidirectional(self.connection, upstream)
except Exception as exc:
logger.exception("Proxy bridge request failed: %s", exc)
try:
self.wfile.write(
b"HTTP/1.1 502 Bad Gateway" + CRLF
+ b"Connection: close" + CRLF
+ b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF
+ b"Bad Gateway"
)
self.wfile.flush()
except OSError:
pass
def _read_headers(self) -> list[tuple[str, str]]:
headers: list[tuple[str, str]] = []
while True:
line = self.rfile.readline(BUFFER_SIZE)
if line in {CRLF, b"\n", b""}:
break
decoded = line.decode("iso-8859-1")
if ":" not in decoded:
continue
name, value = decoded.split(":", 1)
headers.append((name.strip(), value.strip()))
return headers
@staticmethod
def _parse_connect_target(target: str) -> tuple[str, int]:
if target.startswith("["):
end = target.find("]")
if end == -1 or len(target) <= end + 2 or target[end + 1] != ":":
raise ValueError("Invalid CONNECT target")
host = target[1:end]
port_text = target[end + 2 :]
return host, int(port_text)
host, port_text = target.rsplit(":", 1)
return host, int(port_text)
@staticmethod
def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]:
if target.startswith("http://"):
parts = urlsplit(target)
port = parts.port or 80
path = parts.path or "/"
if parts.query:
path += f"?{parts.query}"
return parts.hostname or "", port, path
if target.startswith("https://"):
raise ValueError("HTTPS absolute-form request must use CONNECT")
host_header = next((value for name, value in headers if name.lower() == "host"), "")
if not host_header:
raise ValueError("Missing Host header")
if ":" in host_header:
host, port_text = host_header.rsplit(":", 1)
return host, int(port_text), target
return host_header, 80, target
def _send_forward_request(
self,
upstream: socket.socket,
method: str,
path: str,
version: str,
headers: list[tuple[str, str]],
) -> None:
filtered_headers: list[tuple[str, str]] = []
hop_by_hop = {
"proxy-connection",
"proxy-authorization",
"connection",
"keep-alive",
"te",
"trailer",
"transfer-encoding",
"upgrade",
}
for name, value in headers:
if name.lower() in hop_by_hop:
continue
filtered_headers.append((name, value))
request_head = [f"{method} {path} {version}\r\n"]
request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers)
request_head.append("\r\n")
upstream.sendall("".join(request_head).encode("iso-8859-1"))
def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes:
transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "")
if "chunked" in transfer_encoding.lower():
return self._read_chunked_request_body()
content_length = next((value for name, value in headers if name.lower() == "content-length"), None)
if not content_length:
return b""
return self.rfile.read(int(content_length))
def _read_chunked_request_body(self) -> bytes:
chunks: list[bytes] = []
while True:
size_line = self.rfile.readline(BUFFER_SIZE)
if not size_line:
raise ConnectionError("Unexpected EOF in chunked request")
size_text = size_line.strip().split(b";", 1)[0]
chunk_size = int(size_text, 16)
chunks.append(size_line)
if chunk_size == 0:
while True:
trailer_line = self.rfile.readline(BUFFER_SIZE)
if not trailer_line:
raise ConnectionError("Unexpected EOF in chunked trailers")
chunks.append(trailer_line)
if trailer_line in {CRLF, b"\n"}:
return b"".join(chunks)
chunk_data = self.rfile.read(chunk_size)
if len(chunk_data) != chunk_size:
raise ConnectionError("Unexpected EOF in chunk body")
chunks.append(chunk_data)
chunk_end = self.rfile.read(2)
if chunk_end != CRLF:
raise ConnectionError("Invalid chunk terminator")
chunks.append(chunk_end)
def main() -> None:
if not SOCKS5_HOST:
raise SystemExit("SOCKS5_PROXY_HOST is required")
logging.basicConfig(
level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(),
format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s",
)
with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server:
logger.info(
"Listening on %s:%s -> socks5://%s:%s (max_workers=%s)",
DEFAULT_LISTEN_HOST,
DEFAULT_LISTEN_PORT,
SOCKS5_HOST,
SOCKS5_PORT,
MAX_WORKERS,
)
server.serve_forever()
if __name__ == "__main__":
main()

2101
iaai_scraper/scraper.py
View File

File diff suppressed because it is too large Load Diff

1
iaai_scraper/storage/__init__.py
View File

@@ -1 +0,0 @@
__all__: list[str] = []

648
iaai_scraper/worker/tasks.py
View File

@@ -1,648 +0,0 @@
# Задачи Celery для синхронизации автомобилей и листинга IAAI.
from concurrent.futures import ThreadPoolExecutor
import logging
from threading import Event, Thread
import time
import uuid
from billiard.exceptions import SoftTimeLimitExceeded
from celery import shared_task
from redis import Redis
from ..core.config import Settings, parse_listing_segments
from ..scraper import IAAIScraper
from ..storage.db import PersistenceService
logger = logging.getLogger("iaai_scraper.worker.tasks")
SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak"
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60
SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task"
def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0):
last_exc: Exception | None = None
for attempt in range(1, attempts + 1):
try:
return func()
except Exception as exc:
last_exc = exc
if attempt >= attempts:
break
delay = base_delay_s * (2 ** (attempt - 1))
logger.warning(
"Operation failed (attempt %d/%d): %s. Retrying in %.1fs",
attempt,
attempts,
exc,
delay,
)
time.sleep(delay)
if last_exc is not None:
raise last_exc
def _run_browser_job(func, *args, **kwargs):
# Браузерный код запускаем в отдельном потоке без активного loop.
# НЕ используем `with` — иначе shutdown(wait=True) заблокирует main thread
# если SoftTimeLimitExceeded прервёт future.result(), а browser thread ещё работает.
settings = Settings()
soft = settings.celery.task_soft_time_limit
hard = settings.celery.task_time_limit
# Таймаут для future.result(): берём hard limit (или soft + 120), чтобы не висеть вечно.
wait_timeout = min(hard, soft + 120) if soft and hard else None
executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser")
future = executor.submit(func, *args, **kwargs)
try:
result = future.result(timeout=wait_timeout)
except SoftTimeLimitExceeded:
# Отпускаем executor без ожидания — thread умрёт когда Celery убьёт процесс (hard limit).
future.cancel()
executor.shutdown(wait=False, cancel_futures=True)
raise
except TimeoutError:
# future.result(timeout=...) вышел по таймауту — browser thread завис.
future.cancel()
executor.shutdown(wait=False, cancel_futures=True)
raise SoftTimeLimitExceeded("Browser thread did not finish within time limit")
except Exception:
executor.shutdown(wait=False, cancel_futures=True)
raise
else:
executor.shutdown(wait=True)
return result
def _sync_listing_lock_ttl_seconds() -> int:
settings = Settings()
soft = settings.celery.task_soft_time_limit
hard = settings.celery.task_time_limit
# Используем clamped hard limit (soft + 120), а не сырой task_time_limit,
# чтобы lock не висел 11 дней при CELERY_TASK_TIME_LIMIT=999999.
effective_hard = min(hard, soft + 120) if soft else hard
return max(effective_hard + 120, 300)
def _get_persistence() -> PersistenceService:
settings = Settings()
persistence = PersistenceService(settings)
def _ping_db() -> None:
with persistence.engine.connect() as conn:
conn.exec_driver_sql("SELECT 1")
_retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0)
return persistence
def _get_redis() -> Redis:
settings = Settings()
redis_client = Redis.from_url(
settings.redis.url,
decode_responses=True,
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
socket_timeout=settings.redis.socket_timeout_seconds,
health_check_interval=settings.redis.health_check_interval_seconds,
retry_on_timeout=True,
)
def _ping_redis() -> None:
redis_client.ping()
_retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0)
return redis_client
def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
try:
acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
if acquired:
return True
# Автовосстановление: если lock завис без TTL, считаем stale и пересоздаём.
ttl = redis_client.ttl(key)
if ttl is not None and ttl < 0:
logger.warning("Detected stale lock without TTL, removing: %s", key)
redis_client.delete(key)
return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
return False
except Exception as exc:
logger.warning("Failed to acquire lock %s", key, exc_info=True)
return False
def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None:
try:
refreshed = redis_client.eval(
"""
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
end
return 0
""",
1,
key,
owner_token,
int(ttl_seconds),
)
return bool(refreshed)
except Exception as exc:
logger.warning("Failed to refresh lock %s", key, exc_info=True)
return None
def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None:
try:
redis_client.eval(
"""
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('DEL', KEYS[1])
end
return 0
""",
1,
key,
owner_token,
)
except Exception as exc:
logger.warning("Failed to release lock %s", key, exc_info=True)
def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool:
try:
inspector = celery_app.control.inspect(timeout=1.0)
snapshots = [
inspector.active() or {},
inspector.reserved() or {},
inspector.scheduled() or {},
]
except Exception:
logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True)
return True
for snapshot in snapshots:
for entries in snapshot.values():
for entry in entries or []:
task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
if task_name != SYNC_LISTING_TASK_NAME:
continue
# Исключаем текущую задачу — она не считается "другой запущенной"
entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "")
if exclude_task_id and entry_id == exclude_task_id:
continue
return True
return False
def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool:
try:
owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY)
if not owner_token:
return False
except Exception:
logger.warning("Failed to read sync listing lock before cleanup", exc_info=True)
return False
if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id):
logger.info("sync_listing lock preserved: active task still detected")
return False
try:
ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY)
redis_client.delete(SYNC_LISTING_LOCK_KEY)
logger.warning(
"Removed orphan sync_listing lock owner=%s ttl=%s after worker restart",
owner_token,
ttl,
)
return True
except Exception:
logger.warning("Failed to clear orphan sync listing lock", exc_info=True)
return False
def _is_full_scan_done(redis_client: Redis) -> bool:
try:
value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY)
except Exception:
logger.warning("Failed to read full scan state", exc_info=True)
return False
return str(value or "").strip() == "1"
def _set_full_scan_done(redis_client: Redis, done: bool) -> None:
try:
redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0")
except Exception:
logger.warning("Failed to persist full scan state", exc_info=True)
def _load_last_completed_segment(redis_client: Redis) -> int | None:
# Segment-only checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
try:
raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY)
except Exception:
logger.warning("Failed to read sync listing checkpoint", exc_info=True)
return None
if raw is None:
return None
try:
value = int(str(raw).strip())
except (TypeError, ValueError):
logger.warning("Invalid sync listing checkpoint value %r; clearing", raw)
_clear_sync_checkpoint(redis_client)
return None
if value < 0:
_clear_sync_checkpoint(redis_client)
return None
return value
def _save_last_completed_segment(redis_client: Redis, segment_index: int) -> None:
try:
redis_client.set(
SYNC_LISTING_CHECKPOINT_KEY,
str(int(segment_index)),
ex=SYNC_LISTING_CHECKPOINT_TTL_SECONDS,
)
except Exception:
logger.warning("Failed to save sync listing checkpoint", exc_info=True)
def _clear_sync_checkpoint(redis_client: Redis) -> None:
try:
redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY)
except Exception:
logger.warning("Failed to clear sync listing checkpoint", exc_info=True)
def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool:
try:
return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds))))
except Exception:
logger.warning("Failed to set follow-up pending flag", exc_info=True)
return True
def _clear_followup_pending(redis_client: Redis) -> None:
try:
redis_client.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
except Exception:
logger.warning("Failed to clear follow-up pending flag", exc_info=True)
def _bump_bootstrap_failure_streak(
redis_client: Redis,
*,
reason: str,
) -> tuple[int, bool]:
try:
streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY))
redis_client.expire(
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
)
except Exception:
logger.warning("Failed to update bootstrap failure streak", exc_info=True)
return 0, True
should_enqueue = streak < SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
if should_enqueue:
logger.warning(
"Bootstrap failure streak %d/%d recorded (reason=%s)",
streak,
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT,
reason,
)
else:
logger.error(
"Bootstrap follow-up circuit breaker opened after %d consecutive failures (reason=%s)",
streak,
reason,
)
return streak, should_enqueue
def _clear_bootstrap_failure_streak(redis_client: Redis) -> None:
try:
redis_client.delete(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)
except Exception:
logger.warning("Failed to clear bootstrap failure streak", exc_info=True)
def _start_lock_heartbeat(
redis_client: Redis,
key: str,
owner_token: str,
ttl_seconds: int,
) -> tuple[Event, Thread]:
stop_event = Event()
interval_seconds = max(5.0, min(30.0, ttl_seconds / 3))
def _heartbeat() -> None:
while not stop_event.wait(interval_seconds):
refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds)
if refreshed is False:
logger.warning("Lost sync_listing lock ownership for %s", owner_token)
return
thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True)
thread.start()
return stop_event, thread
@shared_task(
name="iaai_scraper.worker.tasks.sync_vehicle_task",
bind=True,
max_retries=2,
default_retry_delay=30,
acks_late=True,
)
def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
# Скрапинг и upsert одного автомобиля.
persistence = _get_persistence()
persistence.create_tables()
try:
def _job():
with IAAIScraper() as scraper:
return scraper.sync_vehicle(vehicle_url, lane=lane)
result = _run_browser_job(_job)
logger.info("sync_vehicle_task completed: %s", vehicle_url)
return {
"status": "success",
"vehicle_url": vehicle_url,
"db_action": result.get("db_action"),
"images_upserted": result.get("images_upserted", 0),
}
except Exception as exc:
logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True)
raise self.retry(exc=exc)
@shared_task(
name="iaai_scraper.worker.tasks.sync_listing_task",
bind=True,
max_retries=3,
default_retry_delay=120,
acks_late=True,
)
def sync_listing_task(
self,
make: str | None = None,
model: str | None = None,
lane: str = "iaai_cars",
limit: int | None = None,
only_new: bool | None = None,
):
# Полный цикл: листинг + sync всех найденных машин.
persistence = _get_persistence()
persistence.create_tables()
task_id = self.request.id or "unknown"
owner_token = f"{task_id}:{uuid.uuid4().hex}"
redis_client = _get_redis()
lock_acquired = False
lock_ttl = _sync_listing_lock_ttl_seconds()
heartbeat_stop: Event | None = None
heartbeat_thread: Thread | None = None
force_bootstrap_full_scan = False
def _enqueue_bootstrap_followup(
reason: str,
delay_seconds: int = 5,
*,
count_as_failure: bool = False,
) -> None:
flag_ttl = max(lock_ttl, delay_seconds + 300)
if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl):
logger.info(
"Bootstrap follow-up already pending; skip enqueue (reason=%s)",
reason,
)
return
if count_as_failure:
_, should_enqueue = _bump_bootstrap_failure_streak(redis_client, reason=reason)
if not should_enqueue:
_clear_followup_pending(redis_client)
return
else:
_clear_bootstrap_failure_streak(redis_client)
try:
self.app.send_task(
"iaai_scraper.worker.tasks.sync_listing_task",
kwargs={
"make": make,
"model": model,
"lane": lane,
"limit": limit,
"only_new": only_new,
},
queue="scraping",
countdown=max(0, int(delay_seconds)),
)
logger.info(
"Bootstrap follow-up sync queued in %ss (reason=%s)",
delay_seconds,
reason,
)
except Exception:
_clear_followup_pending(redis_client)
logger.warning("Failed to enqueue bootstrap follow-up sync", exc_info=True)
lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
if not lock_acquired:
orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id)
if orphan_cleared:
lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
if not lock_acquired:
logger.info("sync_listing_task skipped: another sync is already running")
return {
"status": "skipped",
"reason": "sync_already_running",
"task_id": task_id,
}
try:
full_scan_done_before_run = _is_full_scan_done(redis_client)
force_bootstrap_full_scan = not full_scan_done_before_run
effective_limit = None if force_bootstrap_full_scan else limit
effective_only_new = False if force_bootstrap_full_scan else only_new
# Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
# Используется только во время bootstrap для пропуска уже обработанных сегментов.
# Никаких page-level resume — внутри сегмента всегда стартуем с page 1.
last_completed_segment: int | None = None
if force_bootstrap_full_scan:
last_completed_segment = _load_last_completed_segment(redis_client)
else:
# После завершения bootstrap чекпоинт не нужен никогда.
_clear_sync_checkpoint(redis_client)
if force_bootstrap_full_scan:
logger.info(
"Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run",
)
logger.info(
"sync_listing options: only_new=%s, limit=%s",
effective_only_new,
effective_limit,
)
_clear_followup_pending(redis_client)
heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
redis_client,
SYNC_LISTING_LOCK_KEY,
owner_token,
lock_ttl,
)
self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
# Определяем сегменты из конфига.
settings = Settings()
segments = parse_listing_segments(settings.listing.listing_segments_json)
use_segmented = bool(segments) and make is None and model is None
resume_from_segment = 0
if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None:
resume_from_segment = max(0, last_completed_segment + 1)
if resume_from_segment >= len(segments):
# Все сегменты уже пройдены — чекпоинт устарел, начинаем заново.
logger.info(
"Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0",
last_completed_segment, len(segments),
)
resume_from_segment = 0
_clear_sync_checkpoint(redis_client)
elif resume_from_segment > 0:
logger.warning(
"Resuming segmented bootstrap from segment=%d (last completed=%d)",
resume_from_segment, last_completed_segment,
)
def _job():
with IAAIScraper() as scraper:
if use_segmented:
return scraper.sync_listing_segmented(
segments=segments,
lane=lane,
only_new=effective_only_new,
start_segment=resume_from_segment,
start_page=1,
progress_callback=(
(lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx))
if force_bootstrap_full_scan else None
),
)
return scraper.sync_listing(
make=make,
model=model,
lane=lane,
limit=effective_limit,
only_new=effective_only_new,
)
result = _run_browser_job(_job)
if force_bootstrap_full_scan:
bootstrap_completed = bool(result.get("full_scan_completed"))
if bootstrap_completed:
_set_full_scan_done(redis_client, True)
_clear_sync_checkpoint(redis_client)
_clear_bootstrap_failure_streak(redis_client)
logger.info("Bootstrap full scan completed; hourly schedule continues")
else:
_set_full_scan_done(redis_client, False)
listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {}
had_progress = any(
int(result.get(key) or 0) > 0
for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered")
) or int(listing_payload.get("vehicles_collected") or 0) > 0
count_as_failure = str(result.get("status") or "") == "failed" and not had_progress
logger.info("Bootstrap full scan not complete yet; queuing immediate continuation")
_enqueue_bootstrap_followup(
"bootstrap_not_completed",
count_as_failure=count_as_failure,
)
else:
_clear_sync_checkpoint(redis_client)
summary = {
"task_id": task_id,
"run_id": result.get("run_id"),
"status": result.get("status", "success"),
"cars_upserted": result.get("cars_upserted", 0),
"cars_failed": result.get("cars_failed", 0),
"images_upserted": result.get("images_upserted", 0),
"skipped_existing": result.get("skipped_existing", 0),
"elapsed_seconds": result.get("elapsed_seconds"),
"failures_count": len(result.get("failures") or []),
}
logger.info(
"sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d",
summary["status"],
summary["cars_upserted"],
summary["cars_failed"],
summary["failures_count"],
)
return summary
except SoftTimeLimitExceeded:
logger.warning(
"sync_listing_task soft timeout exceeded — partial progress already saved to DB"
)
if force_bootstrap_full_scan:
_set_full_scan_done(redis_client, False)
# SoftTimeLimit считаем failure для circuit breaker:
# если сегмент систематически не укладывается в time limit,
# нельзя крутить followup бесконечно.
_enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=True)
# Partial progress уже записан в БД через finish_sync_run.
# Не retry — следующий запуск продолжит обработку по расписанию.
return {
"status": "timed_out",
"task_id": task_id,
"reason": "soft_time_limit_exceeded",
"note": "partial progress saved to DB; bootstrap continuation queued",
}
except Exception as exc:
logger.error("sync_listing_task failed: %s", exc, exc_info=True)
# Retry только на не-таймаутные ошибки (сеть, БД, браузер).
try:
if force_bootstrap_full_scan:
_set_full_scan_done(redis_client, False)
raise self.retry(exc=exc, countdown=5)
raise self.retry(exc=exc)
except self.MaxRetriesExceededError:
logger.error("sync_listing_task max retries exceeded, giving up")
if force_bootstrap_full_scan:
_set_full_scan_done(redis_client, False)
_enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True)
return {
"status": "failed",
"task_id": task_id,
"error": str(exc),
}
finally:
if heartbeat_stop is not None:
heartbeat_stop.set()
if heartbeat_thread is not None:
heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
if lock_acquired:
_release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token)

0
iaai_scraper/__init__.py → openlane_scraper/__init__.py
View File

0
iaai_scraper/api/__init__.py → openlane_scraper/api/__init__.py
View File

4
iaai_scraper/api/app.py → openlane_scraper/api/app.py
View File

@@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
_settings = settings or Settings()
app = FastAPI(
title="IAAI Scraper API",
description="REST API для управления задачами скрапинга IAAI и просмотра данных",
title="OpenLane Scraper API",
description="REST API для управления задачами скрапинга OpenLane и просмотра данных",
version="1.0.0",
lifespan=lifespan,
)

0
iaai_scraper/api/deps.py → openlane_scraper/api/deps.py
View File

0
iaai_scraper/api/routes/__init__.py → openlane_scraper/api/routes/__init__.py
View File

0
iaai_scraper/api/routes/cars.py → openlane_scraper/api/routes/cars.py
View File

4
iaai_scraper/api/routes/health.py → openlane_scraper/api/routes/health.py
View File

@@ -9,7 +9,7 @@ from ..deps import get_persistence
from ...storage.db import PersistenceService
router = APIRouter()
logger = logging.getLogger("iaai_scraper.api.health")
logger = logging.getLogger("openlane_scraper.api.health")
@router.get("/health")
@@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
return {
"status": "ok" if db_ok else "degraded",
"service": "iaai-scraper-api",
"service": "openlane-scraper-api",
"database": "connected" if db_ok else "unavailable",
}

36
iaai_scraper/api/routes/tasks.py → openlane_scraper/api/routes/tasks.py
View File

@@ -8,53 +8,30 @@ from ..deps import get_persistence
from ...storage.db import PersistenceService
from ...storage.models import SyncRun
from ...worker.celery_app import celery_app
from ...worker.tasks import sync_vehicle_task, sync_listing_task
from ...worker.tasks import sync_listing_task
router = APIRouter()
class SyncVehicleRequest(BaseModel):
vehicle_url: str
lane: str = "iaai"
class SyncListingRequest(BaseModel):
make: str | None = None
model: str | None = None
lane: str = "iaai_cars"
lane: str = "openlane_marketplace"
limit: int | None = None
only_new: bool | None = None
@router.post("/tasks/sync-vehicle")
def start_sync_vehicle(
body: SyncVehicleRequest,
):
# Запустить задачу скрапинга одного автомобиля через Celery
result = sync_vehicle_task.apply_async(
kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
queue="scraping",
)
return {
"task_id": result.id,
"status": "queued",
"vehicle_url": body.vehicle_url,
}
max_pages: int | None = None
concurrency: int | None = None
@router.post("/tasks/sync-listing")
def start_sync_listing(
body: SyncListingRequest,
):
# Запустить задачу полного цикла листинга через Celery
result = sync_listing_task.apply_async(
kwargs={
"make": body.make,
"model": body.model,
"lane": body.lane,
"limit": body.limit,
"only_new": body.only_new,
"max_pages": body.max_pages,
"concurrency": body.concurrency,
},
queue="scraping",
)
@@ -90,7 +67,6 @@ def list_sync_runs(
per_page: int = Query(20, ge=1, le=100),
persistence: PersistenceService = Depends(get_persistence),
):
# История запусков синхронизации
with persistence.session_scope() as session:
total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
offset = (page - 1) * per_page

3
openlane_scraper/browser/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from .factory import BrowserFactory
__all__ = ["BrowserFactory"]

23
iaai_scraper/browser/factory.py → openlane_scraper/browser/factory.py
View File

@@ -11,7 +11,7 @@ except ImportError:
from ..core.config import Settings
logger = logging.getLogger("iaai_scraper.browser")
logger = logging.getLogger("openlane_scraper.browser")
def _build_init_script() -> str:
@@ -72,12 +72,12 @@ class BrowserFactory:
# Выбор движка браузера.
engine = self.settings.browser_engine.strip().lower()
if engine == "auto":
# Для IAAI в headless-режиме Chromium со stealth-скриптами
# В headless-режиме Chromium со stealth-скриптами
# значительно стабильнее Firefox по anti-bot.
return "chromium"
if engine in ("firefox", "chromium"):
return engine
logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
logger.warning("Unknown OPENLANE_BROWSER_ENGINE=%r, falling back to auto", engine)
return "chromium"
def create_browser(self, playwright: Playwright) -> Browser:
@@ -136,7 +136,7 @@ class BrowserFactory:
logger.warning("Chrome channel launch failed, falling back to Chromium")
return playwright.chromium.launch(**launch_kwargs)
def create_context(self, browser: Browser) -> BrowserContext:
def create_context(self, browser: Browser, storage_state_path: str | None = None) -> BrowserContext:
viewport = random.choice(self.settings.fingerprint.viewport_presets)
timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
color_scheme = random.choice(["light", "dark"])
@@ -178,6 +178,9 @@ class BrowserFactory:
"Sec-CH-UA-Platform": '"Windows"',
}
if storage_state_path:
ctx_kwargs["storage_state"] = storage_state_path
context = browser.new_context(**ctx_kwargs)
context.set_default_timeout(self.settings.default_timeout_ms)
context.set_default_navigation_timeout(self.settings.default_timeout_ms)
@@ -193,12 +196,20 @@ class BrowserFactory:
@staticmethod
def enable_resource_blocking(page) -> None:
# Блокируем тяжёлые ресурсы для ускорения загрузки страниц.
BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
# Блокируем всё кроме document и XHR/fetch — минимальный след.
BLOCKED_TYPES = {"image", "stylesheet", "font", "media", "websocket", "eventsource", "manifest", "other"}
BLOCKED_URL_PATTERNS = (
"google-analytics", "googletagmanager", "facebook.net",
"doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
"analytics", "tracking", "adservice",
# OpenLane-трекеры из сетевого лога.
"bugsnag", "sessions.bugsnag",
"intercom", "widget.intercom",
"pusher", "sockjs",
"segment", "cdn.segment",
"ap3.com", "ap3c.com",
"onetrust", "optanon", "cookielaw",
"fullstory", "sentry",
)
def _handle_route(route):

92
openlane_scraper/cli.py Normal file
View File

@@ -0,0 +1,92 @@
import argparse
from pathlib import Path
from .core.config import Settings
from .core.utils import save_to_json
from .openlane import OpenLaneScrapeRunner
from .scraper import OpenLaneScraper
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="OpenLane scraper CLI")
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
subparsers = parser.add_subparsers(dest="command", required=True)
subparsers.add_parser("init-db", help="Create DB tables")
sync_parser = subparsers.add_parser("sync-listing", help="Sync OpenLane marketplace → DB")
sync_parser.add_argument("--lane", default="openlane_marketplace")
sync_parser.add_argument("--limit", type=int, default=None)
sync_parser.add_argument("--only-new", choices=["true", "false"], default=None)
sync_parser.add_argument("--max-pages", type=int, default=None)
sync_parser.add_argument("--concurrency", type=int, default=None)
sync_parser.add_argument("--output", default=str(Path("artifacts/json/openlane_sync_listing.json")))
scrape_parser = subparsers.add_parser("scrape-openlane", help="Scrape OpenLane marketplace to JSONL files")
scrape_parser.add_argument("--max-pages", type=int, default=None)
scrape_parser.add_argument("--concurrency", type=int, default=None)
scrape_parser.add_argument("--resume", action="store_true")
scrape_parser.add_argument("--checkpoint-every-pages", type=int, default=None)
subparsers.add_parser("openlane-login", help="Open interactive OpenLane login and persist browser storage state")
return parser
def main() -> None:
parser = build_parser()
args = parser.parse_args()
runtime_settings: Settings | None = None
if args.headless is not None or args.debug:
runtime_settings = Settings()
if args.headless is not None:
runtime_settings.headless = args.headless == "true"
if args.debug:
runtime_settings.log_level = "DEBUG"
if args.command == "openlane-login":
runner = OpenLaneScrapeRunner(runtime_settings)
storage_state_path = runner.interactive_login()
print(f"OpenLane storage state saved to {Path(storage_state_path).resolve()}")
return
if args.command == "scrape-openlane":
runner = OpenLaneScrapeRunner(runtime_settings)
result = runner.run(
max_pages=args.max_pages,
concurrency=args.concurrency,
resume=args.resume,
checkpoint_every_pages=args.checkpoint_every_pages,
)
print(f"OpenLane JSONL saved to {Path(result.jsonl_path).resolve()}")
print(f"OpenLane aggregated JSON saved to {Path(result.aggregated_path).resolve()}")
print(f"OpenLane checkpoint saved to {Path(result.checkpoint_path).resolve()}")
print(f"OpenLane storage state saved to {Path(result.storage_state_path).resolve()}")
print(
f"OpenLane completed pages={len(result.completed_pages)} failed pages={len(result.failed_pages)} total_records={result.total_records} elapsed={result.elapsed_seconds:.2f}s"
)
return
with OpenLaneScraper(runtime_settings) as scraper:
if args.command == "init-db":
data = scraper.init_db()
print(f"DB initialized: {data}")
return
elif args.command == "sync-listing":
only_new = None if args.only_new is None else args.only_new == "true"
data = scraper.sync_listing(
lane=args.lane,
limit=args.limit,
only_new=only_new,
max_pages=args.max_pages,
concurrency=args.concurrency,
)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
if __name__ == "__main__":
main()

0
iaai_scraper/core/__init__.py → openlane_scraper/core/__init__.py
View File

194
openlane_scraper/core/config.py Normal file
View File

@@ -0,0 +1,194 @@
import os
from dataclasses import dataclass, field
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
TRUE_VALUES = {"1", "true", "yes", "on"}
# Хелперы для чтения env-переменных с приведением типов
def _env_str(name: str, default: str) -> str:
value = os.getenv(name)
return value if value is not None else default
def _env_optional_str(name: str) -> str | None:
value = os.getenv(name)
if value is None:
return None
value = value.strip()
return value or None
def _env_path_str(name: str) -> str | None:
value = _env_optional_str(name)
if value is None:
return None
return str(Path(value).expanduser())
def _env_bool(name: str, default: bool) -> bool:
fallback = "true" if default else "false"
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
def _env_int(name: str, default: int) -> int:
return int(_env_str(name, str(default)).strip())
def _env_float(name: str, default: float) -> float:
return float(_env_str(name, str(default)).strip())
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
@dataclass(slots=True)
class FingerprintConfig:
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/135.0.0.0 Safari/537.36"
)
viewport_presets: tuple[dict[str, int], ...] = (
{"width": 1920, "height": 1080},
{"width": 1600, "height": 900},
{"width": 1536, "height": 864},
{"width": 1440, "height": 900},
{"width": 1366, "height": 768},
)
timezone_candidates: tuple[str, ...] = (
"America/New_York",
"America/Chicago",
"America/Los_Angeles",
)
locale: str = "en-US"
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
# --- Конфиг OpenLane (auth, paging, throttle, retry, checkpoint/output) ---
@dataclass(slots=True)
class OpenLaneConfig:
sign_in_url: str = _env_str("OPENLANE_SIGN_IN_URL", "https://app.openlane.com/sign_in")
search_url: str = _env_str("OPENLANE_SEARCH_URL", "https://app.openlane.com/search?tab=marketplace")
api_search_url: str = _env_str("OPENLANE_API_SEARCH_URL", "https://app.openlane.com/api/v4/search")
username: str = _env_str("OPENLANE_USERNAME", "")
password: str = _env_str("OPENLANE_PASSWORD", "")
source_tab: str = _env_str("OPENLANE_SOURCE_TAB", "marketplace")
sale_types: str = _env_str("OPENLANE_SALE_TYPES", "marketplace")
page_size: int = _env_int("OPENLANE_PAGE_SIZE", 30)
max_pages: int = _env_int("OPENLANE_MAX_PAGES", 512)
concurrency: int = _env_int("OPENLANE_CONCURRENCY", 4)
throttle_min_seconds: float = _env_float("OPENLANE_THROTTLE_MIN_SECONDS", 0.3)
throttle_max_seconds: float = _env_float("OPENLANE_THROTTLE_MAX_SECONDS", 0.8)
retry_schedule_seconds: tuple[float, ...] = tuple(
float(part.strip())
for part in _env_str("OPENLANE_RETRY_SCHEDULE_SECONDS", "2,5,10").split(",")
if part.strip()
) or (2.0, 5.0, 10.0)
checkpoint_every_pages: int = _env_int("OPENLANE_CHECKPOINT_EVERY_PAGES", 10)
output_dir: str = _env_str("OPENLANE_OUTPUT_DIR", "artifacts/openlane")
jsonl_output: str = _env_str("OPENLANE_JSONL_OUTPUT", "artifacts/openlane/openlane_search.jsonl")
aggregated_output: str = _env_str("OPENLANE_AGGREGATED_OUTPUT", "artifacts/openlane/openlane_search_aggregated.json")
checkpoint_file: str = _env_str("OPENLANE_CHECKPOINT_FILE", "artifacts/openlane/openlane_checkpoint.json")
storage_state_file: str = _env_str("OPENLANE_STORAGE_STATE_FILE", "artifacts/openlane/openlane_storage_state.json")
persist_storage_state: bool = _env_bool("OPENLANE_PERSIST_STORAGE_STATE", True)
progress_log_every_pages: int = _env_int("OPENLANE_PROGRESS_LOG_EVERY_PAGES", 10)
request_timeout_ms: int = _env_int("OPENLANE_REQUEST_TIMEOUT_MS", 45000)
max_cars_limit: int = _env_int("OPENLANE_MAX_CARS_LIMIT", 20)
refresh_token: str = _env_str("OPENLANE_REFRESH_TOKEN", "")
okta_client_id: str = _env_str("OPENLANE_OKTA_CLIENT_ID", "")
okta_token_endpoint: str = _env_str(
"OPENLANE_OKTA_TOKEN_ENDPOINT",
"https://okta.iam.karglobal.com/oauth2/default/v1/token",
)
okta_redirect_uri: str = _env_str("OPENLANE_OKTA_REDIRECT_URI", "https://app.openlane.com/sign_in")
okta_timeout_seconds: int = _env_int("OPENLANE_OKTA_TIMEOUT_SECONDS", 15)
# --- Конфиг PostgreSQL (URL, пул соединений, pool_recycle) ---
@dataclass(slots=True)
class DatabaseConfig:
url: str = _env_str("OPENLANE_DATABASE_URL", "postgresql+psycopg2://openlane:openlane@localhost:5432/openlane_scraper")
echo: bool = _env_bool("OPENLANE_DATABASE_ECHO", False)
pool_size: int = _env_int("OPENLANE_DATABASE_POOL_SIZE", 5)
max_overflow: int = _env_int("OPENLANE_DATABASE_MAX_OVERFLOW", 10)
pool_recycle_seconds: int = _env_int("OPENLANE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
auto_create_tables: bool = _env_bool("OPENLANE_DATABASE_AUTO_CREATE_TABLES", False)
# --- Конфиг Redis (URL для Celery broker) ---
@dataclass(slots=True)
class RedisConfig:
url: str = _env_str("OPENLANE_REDIS_URL", "redis://localhost:6379/0")
socket_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
socket_connect_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
health_check_interval_seconds: int = _env_int("OPENLANE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@dataclass(slots=True)
class CeleryConfig:
broker_url: str = _env_str("CELERY_BROKER_URL", "")
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
# --- Конфиг прокси (server, username, password) ---
@dataclass(slots=True)
class ProxyConfig:
server: str | None = _env_optional_str("OPENLANE_PROXY_SERVER")
username: str | None = _env_optional_str("OPENLANE_PROXY_USERNAME")
password: str | None = _env_optional_str("OPENLANE_PROXY_PASSWORD")
@property
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
if self.username:
result["username"] = self.username
if self.password:
result["password"] = self.password
return result
# Главный объект настроек: собирает все блоки конфигурации
@dataclass(slots=True)
class Settings:
default_timeout_ms: int = _env_int("OPENLANE_TIMEOUT_MS", 45000)
headless: bool = _env_bool("OPENLANE_HEADLESS", True)
browser_engine: str = _env_str("OPENLANE_BROWSER_ENGINE", "auto")
log_level: str = _env_str("OPENLANE_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("OPENLANE_LOG_FILE")
enable_trace_id_logs: bool = _env_bool("OPENLANE_ENABLE_TRACE_ID_LOGS", True)
runtime_config_file: str | None = _env_path_str("OPENLANE_RUNTIME_CONFIG_FILE")
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
openlane: OpenLaneConfig = field(default_factory=OpenLaneConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
# Глобальный синглтон — используется по умолчанию во всех модулях.
settings = Settings()

0
iaai_scraper/core/logs.py → openlane_scraper/core/logs.py
View File

379
openlane_scraper/core/runtime_config.py Normal file
View File

@@ -0,0 +1,379 @@
"""Runtime config: загрузка и применение runtime_config.json.
Позволяет менять параметры sync и фильтры машин без перезапуска сервиса.
Файл перечитывается при каждом запуске sync_listing.
"""
from __future__ import annotations
import json
import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
logger = logging.getLogger("openlane_scraper.core.runtime_config")
@dataclass(slots=True)
class SyncConfig:
name: str | None = None
ids_initial_size: int | None = None
ids_next_size: int | None = None
ids_max_pages: int | None = None
condition_check_enabled: bool | None = None
lane: str = "openlane_marketplace"
only_new: bool = False
limit: int | None = None
@dataclass(slots=True)
class ListingConfig:
make: str | None = None
model: str | None = None
@dataclass(slots=True)
class RangeFilter:
min: int | None = None
max: int | None = None
@dataclass(slots=True)
class FlagFilters:
damaged_only: bool | None = None
run_and_drive: bool | None = None
@dataclass(slots=True)
class FieldFilters:
brands: list[str] = field(default_factory=list)
models: list[str] = field(default_factory=list)
years: list[int] = field(default_factory=list)
body_types: list[str] = field(default_factory=list)
colors: list[str] = field(default_factory=list)
drives: list[str] = field(default_factory=list)
gearboxes: list[str] = field(default_factory=list)
locations: list[str] = field(default_factory=list)
@dataclass(slots=True)
class Filters:
price: RangeFilter = field(default_factory=RangeFilter)
mileage: RangeFilter = field(default_factory=RangeFilter)
flags: FlagFilters = field(default_factory=FlagFilters)
include: FieldFilters = field(default_factory=FieldFilters)
exclude: FieldFilters = field(default_factory=FieldFilters)
# Legacy flat fields (backward compatibility).
brands: list[str] = field(default_factory=list)
models: list[str] = field(default_factory=list)
years: list[int] = field(default_factory=list)
body_types: list[str] = field(default_factory=list)
colors: list[str] = field(default_factory=list)
drives: list[str] = field(default_factory=list)
gearboxes: list[str] = field(default_factory=list)
locations: list[str] = field(default_factory=list)
exclude_brands: list[str] = field(default_factory=list)
exclude_models: list[str] = field(default_factory=list)
exclude_years: list[int] = field(default_factory=list)
exclude_body_types: list[str] = field(default_factory=list)
exclude_colors: list[str] = field(default_factory=list)
exclude_drives: list[str] = field(default_factory=list)
exclude_gearboxes: list[str] = field(default_factory=list)
exclude_locations: list[str] = field(default_factory=list)
@dataclass(slots=True)
class RuntimeConfig:
sync: SyncConfig = field(default_factory=SyncConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
filters: Filters = field(default_factory=Filters)
def load_runtime_config(path: str | Path | None) -> RuntimeConfig:
"""Загрузить runtime_config.json. Если файл отсутствует — вернуть дефолты."""
if not path:
return RuntimeConfig()
p = Path(path)
if not p.exists():
logger.debug("runtime_config not found at %s — using defaults", p)
return RuntimeConfig()
try:
raw = json.loads(p.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError) as exc:
logger.warning("Failed to parse runtime_config %s: %s — using defaults", p, exc)
return RuntimeConfig()
cfg = RuntimeConfig()
sync_raw = raw.get("sync") or {}
if isinstance(sync_raw, dict):
cfg.sync.name = _opt_str(sync_raw.get("name"))
cfg.sync.ids_initial_size = _opt_int(sync_raw.get("ids_initial_size"))
cfg.sync.ids_next_size = _opt_int(sync_raw.get("ids_next_size"))
cfg.sync.ids_max_pages = _opt_int(sync_raw.get("ids_max_pages"))
cfg.sync.condition_check_enabled = _opt_bool(sync_raw.get("condition_check_enabled"))
cfg.sync.lane = _opt_str(sync_raw.get("lane")) or cfg.sync.lane
only_new = _opt_bool(sync_raw.get("only_new"))
if only_new is not None:
cfg.sync.only_new = only_new
cfg.sync.limit = _opt_int(sync_raw.get("limit"))
listing_raw = raw.get("listing") or {}
if isinstance(listing_raw, dict):
cfg.listing.make = _opt_str(listing_raw.get("make"))
cfg.listing.model = _opt_str(listing_raw.get("model"))
filters_raw = raw.get("filters") or {}
if isinstance(filters_raw, dict):
cfg.filters.price = _parse_range(filters_raw.get("price"))
cfg.filters.mileage = _parse_range(filters_raw.get("mileage"))
cfg.filters.flags = _parse_flags(filters_raw.get("flags"))
include_payload = filters_raw.get("include") if isinstance(filters_raw.get("include"), dict) else filters_raw
exclude_payload = (
filters_raw.get("exclude")
if isinstance(filters_raw.get("exclude"), dict)
else {
"brands": filters_raw.get("exclude_brands"),
"models": filters_raw.get("exclude_models"),
"years": filters_raw.get("exclude_years"),
"body_types": filters_raw.get("exclude_body_types"),
"colors": filters_raw.get("exclude_colors"),
"drives": filters_raw.get("exclude_drives"),
"gearboxes": filters_raw.get("exclude_gearboxes"),
"locations": filters_raw.get("exclude_locations"),
}
)
cfg.filters.include = _parse_fields(include_payload)
cfg.filters.exclude = _parse_fields(exclude_payload)
# Backward-compatible flat aliases.
cfg.filters.brands = list(cfg.filters.include.brands)
cfg.filters.models = list(cfg.filters.include.models)
cfg.filters.years = list(cfg.filters.include.years)
cfg.filters.body_types = list(cfg.filters.include.body_types)
cfg.filters.colors = list(cfg.filters.include.colors)
cfg.filters.drives = list(cfg.filters.include.drives)
cfg.filters.gearboxes = list(cfg.filters.include.gearboxes)
cfg.filters.locations = list(cfg.filters.include.locations)
cfg.filters.exclude_brands = list(cfg.filters.exclude.brands)
cfg.filters.exclude_models = list(cfg.filters.exclude.models)
cfg.filters.exclude_years = list(cfg.filters.exclude.years)
cfg.filters.exclude_body_types = list(cfg.filters.exclude.body_types)
cfg.filters.exclude_colors = list(cfg.filters.exclude.colors)
cfg.filters.exclude_drives = list(cfg.filters.exclude.drives)
cfg.filters.exclude_gearboxes = list(cfg.filters.exclude.gearboxes)
cfg.filters.exclude_locations = list(cfg.filters.exclude.locations)
logger.info(
"Loaded runtime_config from %s: lane=%s, limit=%s",
p,
cfg.sync.lane,
cfg.sync.limit,
)
return cfg
def apply_filters(records: list[dict], filters: Filters) -> list[dict]:
"""Применить runtime-фильтры к списку сырых записей из API."""
if not records:
return records
include = getattr(filters, "include", None)
exclude = getattr(filters, "exclude", None)
include_brands = tuple(getattr(filters, "brands", ()) or getattr(include, "brands", ()))
include_models = tuple(getattr(filters, "models", ()) or getattr(include, "models", ()))
include_years = tuple(getattr(filters, "years", ()) or getattr(include, "years", ()))
include_body_types = tuple(getattr(filters, "body_types", ()) or getattr(include, "body_types", ()))
exclude_brands = tuple(getattr(filters, "exclude_brands", ()) or getattr(exclude, "brands", ()))
exclude_models = tuple(getattr(filters, "exclude_models", ()) or getattr(exclude, "models", ()))
exclude_years = tuple(getattr(filters, "exclude_years", ()) or getattr(exclude, "years", ()))
exclude_body_types = tuple(getattr(filters, "exclude_body_types", ()) or getattr(exclude, "body_types", ()))
price_cfg = getattr(filters, "price", None)
mileage_cfg = getattr(filters, "mileage", None)
flags_cfg = getattr(filters, "flags", None)
price_min = getattr(price_cfg, "min", None)
price_max = getattr(price_cfg, "max", None)
mileage_min = getattr(mileage_cfg, "min", None)
mileage_max = getattr(mileage_cfg, "max", None)
damaged_only = getattr(flags_cfg, "damaged_only", None)
run_and_drive = getattr(flags_cfg, "run_and_drive", None)
result = records
if include_brands:
include_set = {s.casefold() for s in include_brands}
result = [r for r in result if _get_brand(r).casefold() in include_set]
if exclude_brands:
exclude_set = {s.casefold() for s in exclude_brands}
result = [r for r in result if _get_brand(r).casefold() not in exclude_set]
if include_models:
include_set = {s.casefold() for s in include_models}
result = [r for r in result if _get_model(r).casefold() in include_set]
if exclude_models:
exclude_set = {s.casefold() for s in exclude_models}
result = [r for r in result if _get_model(r).casefold() not in exclude_set]
if include_years:
years_set = set(include_years)
result = [r for r in result if _get_year(r) in years_set]
if exclude_years:
years_set = set(exclude_years)
result = [r for r in result if _get_year(r) not in years_set]
if include_body_types:
include_set = {s.casefold() for s in include_body_types}
result = [r for r in result if _get_body_type(r).casefold() in include_set]
if exclude_body_types:
exclude_set = {s.casefold() for s in exclude_body_types}
result = [r for r in result if _get_body_type(r).casefold() not in exclude_set]
if price_min is not None or price_max is not None:
filtered: list[dict] = []
for r in result:
price = _get_price(r)
if price is None:
filtered.append(r)
continue
if price_min is not None and price < price_min:
continue
if price_max is not None and price > price_max:
continue
filtered.append(r)
result = filtered
if mileage_min is not None or mileage_max is not None:
filtered = []
for r in result:
miles = _get_mileage(r)
if miles is None:
filtered.append(r)
continue
if mileage_min is not None and miles < mileage_min:
continue
if mileage_max is not None and miles > mileage_max:
continue
filtered.append(r)
result = filtered
if damaged_only is not None:
result = [r for r in result if bool(r.get("is_damaged")) is damaged_only]
if run_and_drive is not None:
result = [r for r in result if bool(r.get("run_and_drive")) is run_and_drive]
return result
def _parse_range(payload: Any) -> RangeFilter:
payload = payload if isinstance(payload, dict) else {}
return RangeFilter(min=_opt_int(payload.get("min")), max=_opt_int(payload.get("max")))
def _parse_flags(payload: Any) -> FlagFilters:
payload = payload if isinstance(payload, dict) else {}
return FlagFilters(
damaged_only=_opt_bool(payload.get("damaged_only")),
run_and_drive=_opt_bool(payload.get("run_and_drive")),
)
def _parse_fields(payload: Any) -> FieldFilters:
payload = payload if isinstance(payload, dict) else {}
return FieldFilters(
brands=_str_list(payload.get("brands")),
models=_str_list(payload.get("models")),
years=_int_list(payload.get("years")),
body_types=_str_list(payload.get("body_types")),
colors=_str_list(payload.get("colors")),
drives=_str_list(payload.get("drives")),
gearboxes=_str_list(payload.get("gearboxes")),
locations=_str_list(payload.get("locations")),
)
def _opt_int(value: Any) -> int | None:
if value is None:
return None
try:
return int(value)
except (ValueError, TypeError):
return None
def _opt_bool(value: Any) -> bool | None:
if value is None:
return None
if isinstance(value, bool):
return value
if isinstance(value, str):
normalized = value.strip().casefold()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return None
def _opt_str(value: Any) -> str | None:
if value is None:
return None
s = str(value).strip()
return s or None
def _str_list(value: Any) -> list[str]:
if not isinstance(value, list):
return []
return [str(v).strip() for v in value if str(v).strip()]
def _int_list(value: Any) -> list[int]:
if not isinstance(value, list):
return []
result: list[int] = []
for v in value:
iv = _opt_int(v)
if iv is not None:
result.append(iv)
return result
def _get_brand(record: dict[str, Any]) -> str:
return str(record.get("make") or record.get("brand") or "")
def _get_model(record: dict[str, Any]) -> str:
return str(record.get("model") or "")
def _get_body_type(record: dict[str, Any]) -> str:
return str(record.get("body_type") or record.get("body_style") or record.get("vehicle_type") or "")
def _get_year(record: dict[str, Any]) -> int | None:
return _opt_int(record.get("year"))
def _get_price(record: dict[str, Any]) -> int | None:
for key in ("current_high_bid", "buy_now_price", "price", "current_bid", "sale_price"):
value = _opt_int(record.get(key))
if value is not None:
return value
return None
def _get_mileage(record: dict[str, Any]) -> int | None:
return _opt_int(record.get("odometer") or record.get("mileage"))

5
iaai_scraper/core/utils.py → openlane_scraper/core/utils.py
View File

@@ -17,11 +17,6 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
return None
# Регулярные выражения для lot и price.
LOT_RE = re.compile(r"\b(\d{7,10})\b")
PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
# Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
found = []

4
openlane_scraper/openlane/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
from .mapper import map_openlane_record, map_openlane_records
from .runner import OpenLaneScrapeRunner, OpenLaneScrapeResult
__all__ = ["OpenLaneScrapeRunner", "OpenLaneScrapeResult", "map_openlane_record", "map_openlane_records"]

484
openlane_scraper/openlane/auth.py Normal file
View File

@@ -0,0 +1,484 @@
from __future__ import annotations
import base64
import json
import logging
import os
import re
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from playwright.sync_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError
from ..browser.factory import BrowserFactory
from ..core.config import OpenLaneConfig
logger = logging.getLogger("openlane_scraper.openlane.auth")
# Порог предупреждения об истечении refresh_token (дни).
_TOKEN_REFRESH_THRESHOLD_DAYS = 3
_ACCESS_TOKEN_MIN_TTL_SECONDS = 300
class OpenLaneAuthError(RuntimeError):
pass
class OpenLaneAuthenticator:
def __init__(self, config: OpenLaneConfig) -> None:
self.config = config
def validate_credentials(self) -> None:
if not self.config.username or not self.config.password:
raise OpenLaneAuthError(
"OPENLANE_USERNAME and OPENLANE_PASSWORD must be set in the environment"
)
def interactive_login_and_persist(self, context: BrowserContext) -> str:
page = context.new_page()
page.set_default_timeout(self.config.request_timeout_ms)
logger.info("OpenLane interactive login: open %s and complete sign-in manually", self.config.sign_in_url)
page.goto(self.config.sign_in_url, wait_until="domcontentloaded")
self._wait_for_authenticated_session(page)
self._persist_storage_state(context)
logger.info("OpenLane interactive login successful")
return str(Path(self.config.storage_state_file))
def _inject_refresh_token(self, context: BrowserContext) -> None:
# Инъекция refresh_token cookie из env.
token = self.config.refresh_token
if not token:
return
# Проверка: токен уже истёк?
days = self._token_days_remaining(token)
if days is not None and days <= 0:
logger.warning(
"OPENLANE_REFRESH_TOKEN is EXPIRED (%.1f days ago). "
"Will attempt login with username/password.",
abs(days),
)
# Очищаем токен — fallback на login()
self.config.refresh_token = ""
return
if days is not None:
logger.info("refresh_token: %.1f days remaining (expires %s)",
days, datetime.fromtimestamp(
self._decode_jwt_exp(token), tz=timezone.utc # type: ignore[arg-type]
).strftime("%Y-%m-%d %H:%M UTC"))
if days < _TOKEN_REFRESH_THRESHOLD_DAYS:
logger.warning(
"refresh_token expires in %.1f days! "
"Will try to re-login after auth to get a fresh one.",
days,
)
context.add_cookies([
{
"name": "refresh_token",
"value": token,
"domain": ".openlane.com",
"path": "/",
"httpOnly": True,
"secure": True,
"sameSite": "None",
}
])
logger.info("Injected OPENLANE_REFRESH_TOKEN cookie into browser context")
# ------------------------------------------------------------------
# Инспекция и ротация токенов
# ------------------------------------------------------------------
@staticmethod
def _decode_jwt_exp(token: str) -> int | None:
# Декодирование JWT exp (без проверки подписи).
try:
parts = token.split(".")
if len(parts) < 2:
return None
payload_b64 = parts[1]
# Дополнение base64 padding.
payload_b64 += "=" * (-len(payload_b64) % 4)
payload = json.loads(base64.urlsafe_b64decode(payload_b64))
exp = payload.get("exp")
return int(exp) if exp is not None else None
except Exception:
return None
@staticmethod
def _decode_jwt_payload(token: str) -> dict[str, Any]:
# Декодирование JWT payload (без проверки подписи).
try:
parts = token.split(".")
if len(parts) < 2:
return {}
payload_b64 = parts[1]
payload_b64 += "=" * (-len(payload_b64) % 4)
payload = json.loads(base64.urlsafe_b64decode(payload_b64))
return payload if isinstance(payload, dict) else {}
except Exception:
return {}
@staticmethod
def _token_days_remaining(token: str) -> float | None:
# Дней до истечения токена.
exp = OpenLaneAuthenticator._decode_jwt_exp(token)
if exp is None:
return None
now = datetime.now(timezone.utc).timestamp()
return (exp - now) / 86400
@staticmethod
def _token_seconds_remaining(token: str) -> int | None:
# Секунд до истечения токена.
exp = OpenLaneAuthenticator._decode_jwt_exp(token)
if exp is None:
return None
return int(exp - time.time())
@staticmethod
def _extract_cookie_value(context: BrowserContext, name: str) -> str | None:
# Извлекаем значение cookie по имени.
cookies = context.cookies("https://app.openlane.com")
for cookie in cookies:
if cookie.get("name") == name:
value = str(cookie.get("value") or "").strip()
if value:
return value
return None
def _is_access_token_fresh(self, token: str) -> bool:
# Проверяем что access_token ещё жив.
remain = self._token_seconds_remaining(token)
return remain is not None and remain > _ACCESS_TOKEN_MIN_TTL_SECONDS
def _extract_refresh_token_from_cookies(self, context: BrowserContext) -> str | None:
# Извлечение refresh_token из cookies.
return self._extract_cookie_value(context, "refresh_token")
def _extract_access_token_from_cookies(self, context: BrowserContext) -> str | None:
# Извлечение access_token из cookies.
return self._extract_cookie_value(context, "access_token")
def _refresh_access_token_via_okta(self, refresh_token: str) -> tuple[str | None, str | None]:
# Прямой refresh через Okta token endpoint.
if not refresh_token:
return None, None
if not self.config.okta_client_id:
logger.warning("OPENLANE_OKTA_CLIENT_ID is empty, direct refresh disabled")
return None, None
form_data = urlencode(
{
"grant_type": "refresh_token",
"client_id": self.config.okta_client_id,
"redirect_uri": self.config.okta_redirect_uri,
"refresh_token": refresh_token,
}
).encode("utf-8")
req = Request(
self.config.okta_token_endpoint,
data=form_data,
method="POST",
headers={
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "application/json",
},
)
try:
with urlopen(req, timeout=self.config.okta_timeout_seconds) as resp:
payload = json.loads(resp.read().decode("utf-8"))
access_token = str(payload.get("access_token") or "").strip()
new_refresh_token = str(payload.get("refresh_token") or "").strip() or refresh_token
if not access_token:
logger.warning("Okta refresh response has no access_token")
return None, None
logger.info("Okta direct refresh succeeded")
return access_token, new_refresh_token
except HTTPError as exc:
try:
body = exc.read().decode("utf-8", errors="ignore")[:300]
except Exception:
body = ""
logger.warning("Okta direct refresh failed: HTTP %s %s", exc.code, body)
return None, None
except URLError as exc:
logger.warning("Okta direct refresh failed: %s", exc)
return None, None
except Exception as exc:
logger.warning("Okta direct refresh failed: %s", exc)
return None, None
def _persist_access_token_to_storage_state(self, context: BrowserContext, access_token: str) -> None:
# Сохраняем access_token cookie в контекст.
expires = self._decode_jwt_exp(access_token)
if expires is None:
expires = int(time.time()) + 7200
context.add_cookies(
[
{
"name": "access_token",
"value": access_token,
"domain": ".openlane.com",
"path": "/",
"httpOnly": True,
"secure": True,
"sameSite": "None",
"expires": int(expires),
}
]
)
def _maybe_rotate_token(self, context: BrowserContext) -> None:
# Проверка и ротация refresh_token.
new_token = self._extract_refresh_token_from_cookies(context)
if not new_token:
logger.debug("No refresh_token cookie found after auth — nothing to rotate")
return
old_token = self.config.refresh_token or ""
# Проверяем изменился ли токен.
if new_token != old_token:
days = self._token_days_remaining(new_token)
logger.info(
"refresh_token CHANGED (new exp: %.1f days). Persisting to .env",
days if days is not None else -1,
)
self.config.refresh_token = new_token
self._persist_token_to_dotenv(new_token)
return
# Токен не изменился — проверяем срок.
days = self._token_days_remaining(new_token)
if days is not None:
logger.info("refresh_token unchanged, %.1f days remaining", days)
if days < _TOKEN_REFRESH_THRESHOLD_DAYS:
logger.warning(
"⚠ refresh_token expires in %.1f days! "
"Run `openlane-login` or set OPENLANE_USERNAME + OPENLANE_PASSWORD "
"to auto-renew on next sync.",
days,
)
@staticmethod
def _persist_token_to_dotenv(token: str) -> None:
# Запись refresh_token в .env.
env_path = Path(".env")
if not env_path.exists():
# Создаём .env с токеном.
env_path.write_text(
f"OPENLANE_REFRESH_TOKEN={token}\n",
encoding="utf-8",
)
logger.info("Created .env with rotated OPENLANE_REFRESH_TOKEN")
return
content = env_path.read_text(encoding="utf-8")
pattern = re.compile(r"^OPENLANE_REFRESH_TOKEN=.*$", re.MULTILINE)
replacement = f"OPENLANE_REFRESH_TOKEN={token}"
if pattern.search(content):
new_content = pattern.sub(replacement, content)
else:
new_content = content.rstrip("\n") + f"\n{replacement}\n"
env_path.write_text(new_content, encoding="utf-8")
# Обновляем env процесса.
os.environ["OPENLANE_REFRESH_TOKEN"] = token
logger.info("Persisted rotated OPENLANE_REFRESH_TOKEN to .env")
def bootstrap_authenticated_context(self, context: BrowserContext) -> Page:
# Восстановление сессии: storage_state → login → ошибка.
storage_state_path = Path(self.config.storage_state_file)
if storage_state_path.exists():
logger.info("OpenLane: restoring session from %s", storage_state_path)
page = context.new_page()
page.set_default_timeout(self.config.request_timeout_ms)
# Блокируем трекинг/картинки — нужен только fetch().
BrowserFactory.enable_resource_blocking(page)
# Проверка cookies без сетевых запросов.
cookies = context.cookies("https://app.openlane.com")
cookie_names = {c["name"] for c in cookies}
access_token = self._extract_access_token_from_cookies(context)
if access_token and self._is_access_token_fresh(access_token):
logger.info(
"OpenLane session restored: access_token cookie present (%d cookies total)",
len(cookies),
)
# Лёгкий API endpoint для установки origin в браузере.
page.goto(
"https://app.openlane.com/api/_next/time",
wait_until="domcontentloaded",
)
return page
# Пробуем прямой refresh access_token без навигации.
refresh_token = self._extract_refresh_token_from_cookies(context) or self.config.refresh_token
if refresh_token:
new_access_token, new_refresh_token = self._refresh_access_token_via_okta(refresh_token)
if new_access_token:
self._persist_access_token_to_storage_state(context, new_access_token)
if new_refresh_token and new_refresh_token != (self.config.refresh_token or ""):
self.config.refresh_token = new_refresh_token
self._persist_token_to_dotenv(new_refresh_token)
self._persist_storage_state(context)
page.goto(
"https://app.openlane.com/api/_next/time",
wait_until="domcontentloaded",
)
logger.info("OpenLane session restored via direct Okta refresh")
return page
logger.warning("Direct refresh failed, trying sign_in fallback")
# Если refresh_token был только в env, добавляем cookie вручную.
if "refresh_token" not in cookie_names and self.config.refresh_token:
self._inject_refresh_token(context)
cookies = context.cookies("https://app.openlane.com")
cookie_names = {c["name"] for c in cookies}
logger.warning("storage_state exists but no access_token cookie — trying navigation")
# Fallback: SPA обменяет refresh_token на access_token.
if "refresh_token" in cookie_names:
try:
page.goto(self.config.sign_in_url, wait_until="domcontentloaded")
self._wait_for_authenticated_session(page)
self._persist_storage_state(context)
self._maybe_rotate_token(context)
return page
except OpenLaneAuthError:
logger.warning("SPA refresh_token exchange failed")
page.close()
# Нет storage_state — пробуем логин.
if self.config.username and self.config.password:
return self.login(context)
raise OpenLaneAuthError(
"No valid session found. Either:\n"
" 1. Run `python scripts/explore_site.py` to login manually and save storage_state, or\n"
" 2. Set OPENLANE_USERNAME and OPENLANE_PASSWORD in .env"
)
def login(self, context: BrowserContext) -> Page:
self.validate_credentials()
page = context.new_page()
page.set_default_timeout(self.config.request_timeout_ms)
logger.info("OpenLane login: opening sign-in page %s", self.config.sign_in_url)
page.goto(self.config.sign_in_url, wait_until="domcontentloaded")
self._fill_login_form(page)
self._wait_for_authenticated_session(page)
self._persist_storage_state(context)
self._maybe_rotate_token(context)
logger.info("OpenLane login successful")
return page
def _fill_login_form(self, page: Page) -> None:
email_selectors = [
'input[name="username"]',
'input[name="user[login]"]',
'input[name="email"]',
'input[name="user[email]"]',
'input[autocomplete="username"]',
'input[type="email"]',
'input[type="text"]',
'#email',
]
password_selectors = [
'input[name="password"]',
'input[name="user[password]"]',
'input[type="password"]',
'#password',
]
submit_selectors = [
'button[type="submit"]',
'input[type="submit"]',
'button:has-text("Sign in")',
'button:has-text("Log in")',
]
email_filled = self._fill_first(page, email_selectors, self.config.username)
password_filled = self._fill_first(page, password_selectors, self.config.password)
if not email_filled or not password_filled:
raise OpenLaneAuthError("OpenLane login form fields were not found")
if not self._click_first(page, submit_selectors):
raise OpenLaneAuthError("OpenLane login submit button was not found")
def _wait_for_authenticated_session(self, page: Page) -> None:
# Ждём пока SPA обменяет refresh_token → access_token.
try:
page.wait_for_url(
lambda url: "/sign_in" not in url.lower(),
timeout=min(self.config.request_timeout_ms, 30000),
)
except PlaywrightTimeoutError:
raise OpenLaneAuthError(
"OpenLane authentication failed: page stayed on sign_in "
"(refresh_token likely invalid or expired)"
)
# Проверяем что не на странице ошибки.
current_url = page.url.lower()
logger.debug("After auth redirect, URL: %s", page.url)
if "/sign_in" in current_url:
raise OpenLaneAuthError("OpenLane authentication failed: still on sign_in page")
# Ждём появления access_token cookie.
for attempt in range(15):
cookies = page.context.cookies("https://app.openlane.com")
cookie_names = {c["name"] for c in cookies}
if "access_token" in cookie_names:
logger.info("OpenLane authenticated: access_token cookie present (attempt %d)", attempt)
return
page.wait_for_timeout(500)
# Нет access_token cookie, но URL не sign_in — продолжаем.
logger.warning(
"access_token cookie not found after redirect, but page is at %s — continuing",
page.url,
)
def _persist_storage_state(self, context: BrowserContext) -> None:
if not self.config.persist_storage_state:
return
storage_state_path = Path(self.config.storage_state_file)
storage_state_path.parent.mkdir(parents=True, exist_ok=True)
context.storage_state(path=str(storage_state_path))
logger.info("OpenLane storage state saved to %s", storage_state_path)
@staticmethod
def _fill_first(page: Page, selectors: list[str], value: str) -> bool:
for selector in selectors:
locator = page.locator(selector)
if locator.count() == 0:
continue
try:
locator.first.fill(value)
return True
except PlaywrightTimeoutError:
continue
return False
@staticmethod
def _click_first(page: Page, selectors: list[str]) -> bool:
for selector in selectors:
locator = page.locator(selector)
if locator.count() == 0:
continue
try:
locator.first.click()
return True
except PlaywrightTimeoutError:
continue
return False

43
openlane_scraper/openlane/checkpoint.py Normal file
View File

@@ -0,0 +1,43 @@
from __future__ import annotations
import json
from dataclasses import asdict, dataclass, field
from pathlib import Path
@dataclass(slots=True)
class OpenLaneCheckpoint:
max_pages: int
completed_pages: list[int] = field(default_factory=list)
failed_pages: list[int] = field(default_factory=list)
total_records: int = 0
last_saved_at: str | None = None
@property
def completed_set(self) -> set[int]:
return set(self.completed_pages)
class OpenLaneCheckpointStore:
def __init__(self, path: str | Path) -> None:
self.path = Path(path)
def load(self, max_pages: int) -> OpenLaneCheckpoint:
if not self.path.exists():
return OpenLaneCheckpoint(max_pages=max_pages)
data = json.loads(self.path.read_text(encoding="utf-8"))
return OpenLaneCheckpoint(
max_pages=int(data.get("max_pages") or max_pages),
completed_pages=sorted({int(page) for page in data.get("completed_pages", [])}),
failed_pages=sorted({int(page) for page in data.get("failed_pages", [])}),
total_records=int(data.get("total_records") or 0),
last_saved_at=data.get("last_saved_at"),
)
def save(self, checkpoint: OpenLaneCheckpoint) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
payload = asdict(checkpoint)
self.path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
encoding="utf-8",
)

376
openlane_scraper/openlane/client.py Normal file
View File

@@ -0,0 +1,376 @@
from __future__ import annotations
import base64
import json
import logging
import random
import time
from dataclasses import dataclass
from typing import Any
from urllib.parse import urlencode
from playwright.sync_api import Page
from ..core.config import OpenLaneConfig
logger = logging.getLogger("openlane_scraper.openlane.client")
class OpenLaneRequestError(RuntimeError):
def __init__(self, message: str, *, status_code: int | None = None) -> None:
super().__init__(message)
self.status_code = status_code
@dataclass(slots=True)
class OpenLanePageResult:
page: int
records: list[dict[str, Any]]
raw_payload: dict[str, Any]
status_code: int
def _decode_access_token(token: str) -> dict[str, Any]:
# Декодирование JWT payload (без проверки подписи).
parts = token.split(".")
if len(parts) < 2:
return {}
payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4)
try:
return json.loads(base64.urlsafe_b64decode(payload_b64))
except Exception:
return {}
class OpenLaneClient:
# HTTP-клиент OpenLane search API через Playwright.
def __init__(self, authenticated_page: Page, config: OpenLaneConfig) -> None:
self.page = authenticated_page
self.config = config
self._user_id: str | None = None
self._dealership_id: str | None = None
self._init_ids_from_cookies()
def _init_ids_from_cookies(self) -> None:
# Извлекаем user_id и dealership_id из access_token.
cookies = self.page.context.cookies("https://app.openlane.com")
for cookie in cookies:
if cookie.get("name") == "access_token":
payload = _decode_access_token(cookie["value"])
self._user_id = str(payload.get("user_id", ""))
on_behalf = payload.get("on_behalf_of", {})
self._dealership_id = str(on_behalf.get("dealership_id", ""))
if self._user_id and self._dealership_id:
logger.info(
"OpenLane client: user_id=%s dealership_id=%s",
self._user_id, self._dealership_id,
)
return
logger.warning("OpenLane client: access_token cookie not found, API requests may fail")
def _build_headers(self) -> dict[str, str]:
# Заголовки как у фронтенда OpenLane.
headers: dict[str, str] = {
"Accept": "application/json, application/vnd.backlotcars.v3",
"application": "webapp",
}
if self._user_id:
headers["x-rbz-user-id"] = self._user_id
if self._dealership_id:
headers["x-rbz-dealership-id"] = self._dealership_id
return headers
def fetch_page(self, page: int) -> OpenLanePageResult:
params = {
"page": page,
"source_tab": self.config.source_tab,
"sale_types": self.config.sale_types,
}
if self.config.page_size > 0:
params["per_page"] = self.config.page_size
self._sleep_with_jitter()
url = f"{self.config.api_search_url}?{urlencode(params)}"
logger.debug("OpenLane fetch page=%s url=%s", page, url)
# fetch() с авторизацией в браузере.
headers_js = json.dumps(self._build_headers())
fetch_result = self.page.evaluate(
"""async ([url, headersJson]) => {
const headers = JSON.parse(headersJson);
const resp = await fetch(url, {
method: 'GET',
credentials: 'include',
headers: headers
});
const text = await resp.text();
return { status: resp.status, body: text };
}""",
[url, headers_js],
)
return self._parse_response(page, fetch_result)
def fetch_vehicle_images(self, vehicle_id: str | int) -> list[dict[str, Any]]:
"""Загружает изображения автомобиля через `/api/vehicles/{id}/images`."""
if vehicle_id is None:
return []
vehicle_id_str = str(vehicle_id).strip()
if not vehicle_id_str:
return []
self._sleep_with_jitter()
url = f"https://app.openlane.com/api/vehicles/{vehicle_id_str}/images"
logger.debug("OpenLane fetch images vehicle_id=%s", vehicle_id_str)
headers_js = json.dumps(self._build_headers())
fetch_result = self.page.evaluate(
"""async ([url, headersJson]) => {
const headers = JSON.parse(headersJson);
const resp = await fetch(url, {
method: 'GET',
credentials: 'include',
headers: headers
});
const text = await resp.text();
return { status: resp.status, body: text };
}""",
[url, headers_js],
)
status_code = int(fetch_result.get("status", 0))
text = str(fetch_result.get("body", ""))
if status_code == 404:
return []
if status_code == 403:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
status_code=status_code,
)
if status_code == 429:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
status_code=status_code,
)
if status_code >= 500:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
status_code=status_code,
)
if status_code == 401:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
status_code=status_code,
)
if status_code >= 400:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
status_code=status_code,
)
if not text.strip():
return []
try:
payload = json.loads(text)
except json.JSONDecodeError as exc:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
) from exc
return self._extract_vehicle_images(payload)
def fetch_vehicle_images_batch(self, vehicle_ids: list[str | int]) -> dict[str, list[dict[str, Any]]]:
"""Пакетно загружает изображения автомобилей через `/api/vehicles/{id}/images`.
Возвращает словарь `vehicle_id -> список image-объектов`.
"""
normalized_ids: list[str] = []
seen: set[str] = set()
for vehicle_id in vehicle_ids:
vehicle_id_str = str(vehicle_id).strip()
if not vehicle_id_str or vehicle_id_str in seen:
continue
seen.add(vehicle_id_str)
normalized_ids.append(vehicle_id_str)
if not normalized_ids:
return {}
self._sleep_with_jitter()
headers_js = json.dumps(self._build_headers())
fetch_results = self.page.evaluate(
"""async ([vehicleIds, headersJson]) => {
const headers = JSON.parse(headersJson);
const tasks = vehicleIds.map(async (vehicleId) => {
try {
const url = `https://app.openlane.com/api/vehicles/${vehicleId}/images`;
const resp = await fetch(url, {
method: 'GET',
credentials: 'include',
headers: headers,
});
const text = await resp.text();
return { vehicleId, status: resp.status, body: text };
} catch (error) {
return { vehicleId, status: 0, body: '', error: String(error) };
}
});
return await Promise.all(tasks);
}""",
[normalized_ids, headers_js],
)
result_map: dict[str, list[dict[str, Any]]] = {}
for item in fetch_results:
vehicle_id_str = str(item.get("vehicleId", "")).strip()
status_code = int(item.get("status", 0))
text = str(item.get("body", ""))
if not vehicle_id_str:
continue
if status_code == 404:
result_map[vehicle_id_str] = []
continue
if status_code == 403:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
status_code=status_code,
)
if status_code == 429:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
status_code=status_code,
)
if status_code >= 500 or status_code == 0:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
status_code=status_code if status_code else 500,
)
if status_code == 401:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
status_code=status_code,
)
if status_code >= 400:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
status_code=status_code,
)
if not text.strip():
result_map[vehicle_id_str] = []
continue
try:
payload = json.loads(text)
except json.JSONDecodeError as exc:
raise OpenLaneRequestError(
f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
) from exc
result_map[vehicle_id_str] = self._extract_vehicle_images(payload)
return result_map
def _parse_response(self, page: int, fetch_result: dict[str, Any]) -> OpenLanePageResult:
# Парсинг ответа fetch(): {status, body}.
status_code = int(fetch_result.get("status", 0))
text = str(fetch_result.get("body", ""))
logger.debug(
"OpenLane response: page=%s status=%s body_len=%d body_preview=%.500s",
page, status_code, len(text), text[:500],
)
if status_code == 403:
raise OpenLaneRequestError(
f"OpenLane page={page} returned 403 Forbidden — possible block, stopping",
status_code=status_code,
)
if status_code == 429:
raise OpenLaneRequestError(
f"OpenLane page={page} returned 429 Too Many Requests — rate limited",
status_code=status_code,
)
if status_code >= 500:
raise OpenLaneRequestError(
f"OpenLane page={page} returned server error {status_code}",
status_code=status_code,
)
if status_code == 401:
raise OpenLaneRequestError(
f"OpenLane page={page} returned 401; session is not authenticated",
status_code=status_code,
)
if status_code >= 400:
raise OpenLaneRequestError(
f"OpenLane page={page} returned unexpected status {status_code}",
status_code=status_code,
)
try:
payload = json.loads(text)
except json.JSONDecodeError as exc:
raise OpenLaneRequestError(f"OpenLane page={page} returned invalid JSON") from exc
records = self._extract_records(payload)
logger.debug(
"OpenLane extracted: page=%s records=%d top_keys=%s",
page, len(records), list(payload.keys())[:10],
)
return OpenLanePageResult(
page=page,
records=records,
raw_payload=payload,
status_code=status_code,
)
@staticmethod
def _extract_records(payload: dict[str, Any]) -> list[dict[str, Any]]:
candidates = [
payload.get("results"),
payload.get("items"),
payload.get("data"),
payload.get("vehicles"),
]
for candidate in candidates:
if isinstance(candidate, list):
return [item for item in candidate if isinstance(item, dict)]
if isinstance(candidate, dict):
nested_candidates = [
candidate.get("results"),
candidate.get("items"),
candidate.get("vehicles"),
]
for nested in nested_candidates:
if isinstance(nested, list):
return [item for item in nested if isinstance(item, dict)]
return []
@staticmethod
def _extract_vehicle_images(payload: Any) -> list[dict[str, Any]]:
candidates: list[Any] = []
if isinstance(payload, dict):
candidates.extend(
[
payload.get("vehicle_images"),
payload.get("images"),
payload.get("data"),
]
)
data = payload.get("data")
if isinstance(data, dict):
candidates.extend([data.get("vehicle_images"), data.get("images")])
elif isinstance(payload, list):
candidates.append(payload)
for candidate in candidates:
if isinstance(candidate, list):
return [item for item in candidate if isinstance(item, dict)]
return []
def _sleep_with_jitter(self) -> None:
low = min(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
high = max(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
time.sleep(random.uniform(low, high))

424
openlane_scraper/openlane/mapper.py Normal file
View File

@@ -0,0 +1,424 @@
"""Маппер: JSON-запись OpenLane API → CarRecord для сохранения в БД."""
from __future__ import annotations
import hashlib
import logging
import re
from datetime import datetime, timezone
from typing import Any
from ..storage.schemas import CarRecord, ImageRecord
logger = logging.getLogger("openlane_scraper.openlane.mapper")
# Маппинг body_type из OpenLane в наш enum.
BODY_MAP: dict[str, str] = {
"sedan": "SEDAN",
"coupe": "COUPE",
"suv": "SUV",
"sport utility": "SUV",
"crossover": "SUV",
"hatchback": "HATCHBACK",
"minivan": "MINIVAN",
"van": "MINIVAN",
"wagon": "STATION_WAGON",
"station wagon": "STATION_WAGON",
"pickup": "PICKUP",
"truck": "TRUCK",
"convertible": "OPEN",
"cabriolet": "OPEN",
"roadster": "OPEN",
"rv": "RV",
"motorhome": "RV",
}
DRIVE_MAP: dict[str, str] = {
"fwd": "FWD",
"front wheel drive": "FWD",
"front-wheel drive": "FWD",
"rwd": "RWD",
"rear wheel drive": "RWD",
"rear-wheel drive": "RWD",
"awd": "4WD",
"4wd": "4WD",
"all wheel drive": "4WD",
"all-wheel drive": "4WD",
"4x4": "4WD",
"2wd": "2WD",
"two wheel drive": "2WD",
}
GEARBOX_MAP: dict[str, str] = {
"automatic": "AT",
"auto": "AT",
"at": "AT",
"manual": "MT",
"mt": "MT",
"cvt": "CVT",
"continuously variable": "CVT",
"electric": "EV",
"ev": "EV",
}
_MILEAGE_RE = re.compile(r"[\d,]+")
_ENGINE_RE = re.compile(r"(\d+\.?\d*)\s*[lL]")
def _safe_str(value: Any, default: str = "") -> str:
if value is None:
return default
return str(value).strip() or default
def _safe_int(value: Any) -> int | None:
if value is None:
return None
try:
cleaned = str(value).replace(",", "").strip()
if not cleaned:
return None
return int(float(cleaned))
except (ValueError, TypeError):
return None
def _normalize_enum(raw: Any, mapping: dict[str, str], default: str = "NA") -> str:
if not raw:
return default
key = str(raw).strip().lower()
return mapping.get(key, default)
def _generate_parser_id(origin_id: str) -> str:
return hashlib.sha256(origin_id.encode()).hexdigest()[:40]
def _generate_slug(year: int | None, brand: str, model: str, origin_id: str) -> str:
parts = []
if year:
parts.append(str(year))
parts.append(brand.lower())
parts.append(model.lower())
parts.append(origin_id.replace(":", "-"))
slug = "-".join(parts)
slug = re.sub(r"[^a-z0-9\-]", "-", slug)
slug = re.sub(r"-+", "-", slug).strip("-")
return slug[:200]
def _build_openlane_origin_id(raw_id: Any) -> str | None:
"""Строит origin_id в формате openlane:id."""
normalized = _safe_str(raw_id)
if not normalized:
return None
if normalized.lower().startswith("openlane:"):
normalized = _safe_str(normalized.split(":", 1)[1])
if not normalized:
return None
return f"openlane:{normalized}"
def _extract_nested(record: dict[str, Any], *keys: str) -> Any:
"""Извлекает значение из вложенного словаря по цепочке ключей."""
obj: Any = record
for key in keys:
if isinstance(obj, dict):
obj = obj.get(key)
else:
return None
return obj
def _extract_images(record: dict[str, Any]) -> list[ImageRecord]:
"""Извлекает изображения из записи OpenLane."""
images: list[ImageRecord] = []
seen_urls: set[str] = set()
raw_images = (
record.get("images")
or record.get("photos")
or record.get("media", {}).get("images")
or record.get("image_urls")
or []
)
if isinstance(raw_images, list):
for idx, img in enumerate(raw_images):
if isinstance(img, str):
url = img.strip()
if url and url not in seen_urls:
seen_urls.add(url)
images.append(ImageRecord(
fullres_image=url,
preview_image=url,
order_index=idx,
))
elif isinstance(img, dict):
fullres = _safe_str(
img.get("full") or img.get("fullres") or img.get("url")
or img.get("original") or img.get("large") or img.get("href")
or img.get("large_resolution_url")
)
preview = _safe_str(
img.get("thumbnail") or img.get("thumb") or img.get("preview")
or img.get("small") or img.get("low_resolution_url")
or fullres
)
if fullres and fullres not in seen_urls:
seen_urls.add(fullres)
images.append(ImageRecord(
fullres_image=fullres,
preview_image=preview,
order_index=idx,
))
# Fallback: одиночное изображение.
if not images:
single = (
record.get("image_url")
or record.get("image")
or record.get("large_resolution_url")
or record.get("low_resolution_url")
or record.get("thumbnail")
or record.get("photo_url")
or _extract_nested(record, "media", "primary")
)
if single and isinstance(single, str) and single.strip():
images.append(ImageRecord(
fullres_image=single.strip(),
preview_image=single.strip(),
order_index=0,
))
return images
def _parse_mileage(raw: Any) -> int:
if raw is None:
return 0
if isinstance(raw, (int, float)):
return max(0, int(raw))
text = str(raw)
match = _MILEAGE_RE.search(text)
if match:
try:
return max(0, int(match.group().replace(",", "")))
except ValueError:
pass
return 0
def _parse_engine_volume_cc(raw: Any) -> int | None:
"""Парсит объём двигателя и возвращает значение в кубических сантиметрах."""
if raw is None:
return None
if isinstance(raw, (int, float)):
value = float(raw)
# Если значение < 20 — скорее всего это литры, конвертируем в cc.
if 0 < value < 20:
return int(value * 1000)
if value >= 100:
return int(value)
return None
text = str(raw)
match = _ENGINE_RE.search(text)
if match:
liters = float(match.group(1))
return int(liters * 1000)
return None
def map_openlane_record(record: dict[str, Any]) -> CarRecord | None:
"""Маппит одну запись из OpenLane API в CarRecord.
Возвращает None, если запись не содержит минимально необходимых данных.
"""
# Извлекаем идентификатор.
raw_id = (
record.get("id")
or record.get("vehicle_id")
or record.get("listing_id")
or record.get("vin")
)
if not raw_id:
logger.debug("Skipping record without id: %s", record.get("vin", "unknown"))
return None
origin_id = _build_openlane_origin_id(raw_id)
if not origin_id:
logger.debug("Skipping record with malformed id: %s", raw_id)
return None
# Бренд и модель — обязательные поля.
brand = _safe_str(
record.get("make")
or record.get("brand")
or record.get("manufacturer")
or _extract_nested(record, "vehicle", "make")
).upper()
model = _safe_str(
record.get("model")
or record.get("model_name")
or _extract_nested(record, "vehicle", "model")
).upper()
if not brand or not model:
logger.debug("Skipping record without brand/model: %s", origin_id)
return None
year = _safe_int(
record.get("year")
or record.get("model_year")
or _extract_nested(record, "vehicle", "year")
)
price = _safe_int(
record.get("price")
or record.get("current_bid")
or record.get("buy_now_price")
or record.get("asking_price")
or record.get("sale_price")
or _extract_nested(record, "pricing", "current")
or _extract_nested(record, "pricing", "buy_now")
)
currency = _safe_str(
record.get("currency")
or record.get("currency_code")
or _extract_nested(record, "pricing", "currency"),
"USD",
).upper()
if currency not in {"JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD"}:
currency = "USD"
mileage = _parse_mileage(
record.get("mileage")
or record.get("odometer")
or record.get("odometer_reading")
or _extract_nested(record, "vehicle", "mileage")
)
color = _safe_str(
record.get("color")
or record.get("exterior_color")
or _extract_nested(record, "vehicle", "color"),
"other",
).lower()
body_type = _normalize_enum(
record.get("body_type")
or record.get("body_style")
or record.get("vehicle_type")
or _extract_nested(record, "vehicle", "body_type"),
BODY_MAP,
"OTHER",
)
drive = _normalize_enum(
record.get("drive_type")
or record.get("drivetrain")
or record.get("drive")
or _extract_nested(record, "vehicle", "drivetrain"),
DRIVE_MAP,
)
gearbox = _normalize_enum(
record.get("transmission")
or record.get("gearbox")
or _extract_nested(record, "vehicle", "transmission"),
GEARBOX_MAP,
)
engine_volume = _parse_engine_volume_cc(
record.get("engine")
or record.get("engine_size")
or record.get("displacement")
or _extract_nested(record, "vehicle", "engine")
)
# URL записи на OpenLane.
origin_url = _safe_str(
record.get("url")
or record.get("listing_url")
or record.get("detail_url")
or record.get("permalink")
)
if not origin_url:
origin_url = f"https://app.openlane.com/vehicles/{raw_id}"
country = _safe_str(
record.get("country")
or record.get("location_country")
or _extract_nested(record, "location", "country"),
"US",
).upper()
if country not in {"JP", "KR", "US", "CA", "NA"}:
country = "US"
is_damaged = bool(
record.get("is_damaged")
or record.get("has_damage")
or record.get("damage_type")
)
vin = _safe_str(record.get("vin") or _extract_nested(record, "vehicle", "vin"))
evaluation = vin if vin else None
selling_type = "AUCTION"
sale_type = _safe_str(record.get("sale_type") or record.get("listing_type")).lower()
if sale_type in {"buy_now", "fixed_price", "stock"}:
selling_type = "STOCK"
elif sale_type in {"tender"}:
selling_type = "TENDER"
images = _extract_images(record)
parser_id = _generate_parser_id(origin_id)
slug = _generate_slug(year, brand, model, origin_id)
return CarRecord(
parser_id=parser_id,
brand=brand,
model=model,
year=year,
price=price,
currency=currency,
mileage=mileage,
country=country,
is_sold=False,
color=color,
drive=drive if drive != "NA" else None,
gearbox=gearbox if gearbox != "NA" else None,
body_type=body_type,
engine_volume=engine_volume,
selling_type=selling_type,
origin="OPENLANE",
origin_url=origin_url,
origin_id=origin_id,
is_damaged=is_damaged,
evaluation=evaluation,
slug=slug,
images=images,
)
def map_openlane_records(records: list[dict[str, Any]]) -> list[CarRecord]:
"""Маппит список записей OpenLane API в список CarRecord.
Пропускает записи без минимально необходимых данных.
"""
result: list[CarRecord] = []
for record in records:
try:
car = map_openlane_record(record)
if car is not None:
result.append(car)
except Exception:
logger.warning(
"Failed to map OpenLane record id=%s",
record.get("id", "unknown"),
exc_info=True,
)
return result

273
openlane_scraper/openlane/runner.py Normal file
View File

@@ -0,0 +1,273 @@
from __future__ import annotations
import logging
import time
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from playwright.sync_api import sync_playwright
from ..browser.factory import BrowserFactory
from ..core.config import Settings
from ..core.logs import set_trace_id, setup_logging
from .auth import OpenLaneAuthenticator
from .checkpoint import OpenLaneCheckpoint, OpenLaneCheckpointStore
from .client import OpenLaneClient, OpenLanePageResult, OpenLaneRequestError
from .writer import OpenLaneResultWriter
logger = logging.getLogger("openlane_scraper.openlane.runner")
@dataclass(slots=True)
class OpenLaneScrapeResult:
jsonl_path: str
aggregated_path: str
checkpoint_path: str
storage_state_path: str
completed_pages: list[int]
failed_pages: list[int]
total_records: int
elapsed_seconds: float
class OpenLaneScrapeRunner:
def __init__(self, runtime_settings: Settings | None = None) -> None:
self.settings = runtime_settings or Settings()
setup_logging(self.settings.log_level, self.settings.log_file)
self.trace_id = f"openlane-{uuid.uuid4().hex[:8]}"
set_trace_id(self.trace_id)
self.openlane = self.settings.openlane
self.browser_factory = BrowserFactory(self.settings)
self.authenticator = OpenLaneAuthenticator(self.openlane)
self.writer = OpenLaneResultWriter(
self.openlane.jsonl_output,
self.openlane.aggregated_output,
)
self.checkpoint_store = OpenLaneCheckpointStore(self.openlane.checkpoint_file)
def run(
self,
*,
max_pages: int | None = None,
concurrency: int | None = None,
resume: bool = False,
checkpoint_every_pages: int | None = None,
) -> OpenLaneScrapeResult:
started_at = time.perf_counter()
max_pages = max_pages or self.openlane.max_pages
concurrency = max(1, min(concurrency or self.openlane.concurrency, 5))
checkpoint_every_pages = checkpoint_every_pages or self.openlane.checkpoint_every_pages
checkpoint = self.checkpoint_store.load(max_pages=max_pages) if resume else OpenLaneCheckpoint(max_pages=max_pages)
checkpoint.max_pages = max_pages
if not resume:
self._reset_outputs()
logger.info(
"Starting OpenLane scrape: max_pages=%s concurrency=%s resume=%s checkpoint_every_pages=%s",
max_pages,
concurrency,
resume,
checkpoint_every_pages,
)
with sync_playwright() as playwright:
browser = self.browser_factory.create_browser(playwright)
try:
auth_pages = []
for worker_index in range(concurrency):
storage_state_path = self.openlane.storage_state_file if Path(self.openlane.storage_state_file).exists() else None
context = self.browser_factory.create_context(browser, storage_state_path=storage_state_path)
auth_page = self.authenticator.bootstrap_authenticated_context(context)
auth_pages.append(auth_page)
logger.info("OpenLane worker session initialized: worker=%s", worker_index + 1)
pending_pages = [
page for page in range(1, max_pages + 1)
if page not in checkpoint.completed_set
]
self._run_workers(auth_pages, pending_pages, checkpoint, checkpoint_every_pages)
finally:
browser.close()
self.checkpoint_store.save(checkpoint)
aggregated = self.writer.finalize(
max_pages=max_pages,
completed_pages=checkpoint.completed_pages,
failed_pages=checkpoint.failed_pages,
)
elapsed_seconds = time.perf_counter() - started_at
logger.info(
"OpenLane scrape finished: completed_pages=%s failed_pages=%s total_records=%s elapsed=%.2fs",
len(set(checkpoint.completed_pages)),
len(set(checkpoint.failed_pages)),
aggregated["total_records"],
elapsed_seconds,
)
return OpenLaneScrapeResult(
jsonl_path=str(Path(self.openlane.jsonl_output)),
aggregated_path=str(Path(self.openlane.aggregated_output)),
checkpoint_path=str(Path(self.openlane.checkpoint_file)),
storage_state_path=str(Path(self.openlane.storage_state_file)),
completed_pages=sorted(set(checkpoint.completed_pages)),
failed_pages=sorted(set(checkpoint.failed_pages)),
total_records=int(aggregated["total_records"]),
elapsed_seconds=elapsed_seconds,
)
def _run_workers(
self,
auth_pages: list,
pending_pages: list[int],
checkpoint: OpenLaneCheckpoint,
checkpoint_every_pages: int,
) -> None:
# Последовательная обработка страниц (sync API — однопоточный).
started_at = time.perf_counter()
for idx, page_num in enumerate(pending_pages):
worker_index = (idx % len(auth_pages)) + 1
auth_page = auth_pages[idx % len(auth_pages)]
try:
result = self._fetch_page_with_retry(auth_page, page_num, worker_index)
self._handle_success(result, checkpoint, started_at)
except Exception as exc:
self._handle_failure(page_num, exc, checkpoint, started_at)
processed_count = len(set(checkpoint.completed_pages)) + len(set(checkpoint.failed_pages))
if processed_count and processed_count % checkpoint_every_pages == 0:
checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat()
self.checkpoint_store.save(checkpoint)
logger.info(
"OpenLane checkpoint saved: processed=%s completed=%s failed=%s",
processed_count,
len(set(checkpoint.completed_pages)),
len(set(checkpoint.failed_pages)),
)
def _fetch_page_with_retry(self, authenticated_page, page: int, worker_index: int) -> OpenLanePageResult:
set_trace_id(f"{self.trace_id}-w{worker_index}-p{page}")
client = OpenLaneClient(authenticated_page, self.openlane)
retry_schedule = self.openlane.retry_schedule_seconds
for attempt in range(len(retry_schedule) + 1):
try:
logger.debug("OpenLane fetch attempt: page=%s worker=%s attempt=%s", page, worker_index, attempt + 1)
return client.fetch_page(page)
except OpenLaneRequestError as exc:
is_retryable = exc.status_code in {403, 429} or (exc.status_code is not None and exc.status_code >= 500)
if not is_retryable or attempt >= len(retry_schedule):
logger.error(
"OpenLane fetch failed permanently: page=%s worker=%s status=%s error=%s",
page,
worker_index,
exc.status_code,
exc,
)
raise
delay = retry_schedule[attempt]
logger.warning(
"OpenLane retry scheduled: page=%s worker=%s status=%s delay=%.1fs attempt=%s",
page,
worker_index,
exc.status_code,
delay,
attempt + 1,
)
time.sleep(delay)
except Exception:
if attempt >= len(retry_schedule):
raise
delay = retry_schedule[attempt]
logger.warning(
"OpenLane transient error retry: page=%s worker=%s delay=%.1fs attempt=%s",
page,
worker_index,
delay,
attempt + 1,
exc_info=True,
)
time.sleep(delay)
raise RuntimeError(f"OpenLane page {page} exhausted retries")
def _handle_success(
self,
result: OpenLanePageResult,
checkpoint: OpenLaneCheckpoint,
started_at: float,
) -> None:
written = self.writer.append_page(result.page, result.records)
checkpoint.completed_pages = sorted(set(checkpoint.completed_pages) | {result.page})
checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) - {result.page})
checkpoint.total_records += written
checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat()
self._log_progress(result.page, checkpoint, started_at, written, None)
def _handle_failure(
self,
page: int,
exc: Exception,
checkpoint: OpenLaneCheckpoint,
started_at: float,
) -> None:
checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) | {page})
checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat()
self._log_progress(page, checkpoint, started_at, 0, exc)
def _log_progress(
self,
page: int,
checkpoint: OpenLaneCheckpoint,
started_at: float,
records_written: int,
exc: Exception | None,
) -> None:
completed = len(set(checkpoint.completed_pages))
failed = len(set(checkpoint.failed_pages))
elapsed_seconds = max(time.perf_counter() - started_at, 0.001)
pages_per_minute = (completed + failed) / elapsed_seconds * 60.0
if exc is None:
logger.info(
"OpenLane progress: page=%s completed=%s failed=%s records=%s total_records=%s speed=%.2f pages/min",
page,
completed,
failed,
records_written,
checkpoint.total_records,
pages_per_minute,
)
else:
logger.error(
"OpenLane page error: page=%s completed=%s failed=%s total_records=%s speed=%.2f pages/min error=%s",
page,
completed,
failed,
checkpoint.total_records,
pages_per_minute,
exc,
)
def interactive_login(self) -> str:
setup_logging(self.settings.log_level, self.settings.log_file)
with sync_playwright() as playwright:
browser = self.browser_factory.create_browser(playwright)
try:
context = self.browser_factory.create_context(browser)
return self.authenticator.interactive_login_and_persist(context)
finally:
browser.close()
def _reset_outputs(self) -> None:
for raw_path in (
self.openlane.jsonl_output,
self.openlane.aggregated_output,
self.openlane.checkpoint_file,
):
path = Path(raw_path)
if path.exists():
path.unlink()

55
openlane_scraper/openlane/writer.py Normal file
View File

@@ -0,0 +1,55 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
class OpenLaneResultWriter:
def __init__(self, jsonl_path: str | Path, aggregated_path: str | Path) -> None:
self.jsonl_path = Path(jsonl_path)
self.aggregated_path = Path(aggregated_path)
def ensure_parent_dirs(self) -> None:
self.jsonl_path.parent.mkdir(parents=True, exist_ok=True)
self.aggregated_path.parent.mkdir(parents=True, exist_ok=True)
def append_page(self, page: int, records: list[dict[str, Any]]) -> int:
self.ensure_parent_dirs()
written = 0
with self.jsonl_path.open("a", encoding="utf-8") as fh:
for record in records:
payload = {
"page": page,
"record": record,
}
fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
written += 1
return written
def finalize(self, *, max_pages: int, completed_pages: list[int], failed_pages: list[int]) -> dict[str, Any]:
self.ensure_parent_dirs()
records: list[dict[str, Any]] = []
if self.jsonl_path.exists():
with self.jsonl_path.open("r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if not line:
continue
item = json.loads(line)
records.append(item)
summary = {
"max_pages": max_pages,
"completed_pages": sorted(completed_pages),
"failed_pages": sorted(failed_pages),
"total_pages_completed": len(set(completed_pages)),
"total_pages_failed": len(set(failed_pages)),
"total_records": len(records),
"items": records,
}
self.aggregated_path.write_text(
json.dumps(summary, ensure_ascii=False, indent=2),
encoding="utf-8",
)
return summary

514
openlane_scraper/scraper.py Normal file
View File

@@ -0,0 +1,514 @@
"""Главный оркестратор скрапинга OpenLane с сохранением в БД.
Использует OpenLane API для получения данных и PersistenceService для записи в PostgreSQL.
Batched upsert, SyncRun-аудит, early-stop при отсутствии новых записей.
"""
from __future__ import annotations
import logging
import time
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from playwright.sync_api import sync_playwright
from .browser.factory import BrowserFactory
from .core.config import Settings
from .core.logs import set_trace_id, setup_logging
from .core.runtime_config import RuntimeConfig, apply_filters, load_runtime_config
from .openlane.auth import OpenLaneAuthenticator
from .openlane.client import OpenLaneClient, OpenLanePageResult, OpenLaneRequestError
from .openlane.mapper import map_openlane_records
from .storage.db import PersistenceService
from .storage.schemas import CarRecord, ImageRecord
logger = logging.getLogger("openlane_scraper.scraper")
class OpenLaneScraper:
"""Оркестратор: OpenLane API → маппинг → DB upsert."""
def __init__(self, runtime_settings: Settings | None = None) -> None:
self.settings = runtime_settings or Settings()
setup_logging(self.settings.log_level, self.settings.log_file)
self.trace_id = f"openlane-sync-{uuid.uuid4().hex[:8]}"
set_trace_id(self.trace_id)
self.openlane_config = self.settings.openlane
self.browser_factory = BrowserFactory(self.settings)
self.authenticator = OpenLaneAuthenticator(self.openlane_config)
self.persistence = PersistenceService(self.settings)
self.runtime_config = load_runtime_config(self.settings.runtime_config_file)
def __enter__(self):
return self
def __exit__(self, *args):
pass
@staticmethod
def _normalize_origin_id(raw_id: Any) -> str | None:
if raw_id is None:
return None
normalized = str(raw_id).strip()
if not normalized:
return None
if normalized.lower().startswith("openlane:"):
normalized = normalized.split(":", 1)[1].strip()
if not normalized:
return None
return f"openlane:{normalized}"
@staticmethod
def _build_image_records(images_payload: list[dict[str, Any]]) -> list[ImageRecord]:
images: list[ImageRecord] = []
seen: set[str] = set()
for idx, img in enumerate(images_payload):
fullres = str(
img.get("url")
or img.get("large_resolution_url")
or img.get("image_large")
or img.get("image")
or img.get("full")
or img.get("fullres")
or ""
).strip()
if not fullres or fullres in seen:
continue
preview = str(
img.get("low_resolution_url")
or img.get("medium_resolution_url")
or img.get("thumbnail_url")
or img.get("thumbnail")
or img.get("thumb")
or img.get("image")
or fullres
).strip()
seen.add(fullres)
images.append(
ImageRecord(
fullres_image=fullres,
preview_image=preview or fullres,
order_index=idx,
)
)
return images
@staticmethod
def _merge_image_records(existing: list[ImageRecord], fetched: list[ImageRecord]) -> list[ImageRecord]:
merged: list[ImageRecord] = []
seen: set[str] = set()
for source in (fetched, existing):
for image in source:
key = image.fullres_image.strip()
if not key or key in seen:
continue
seen.add(key)
merged.append(image)
for idx, image in enumerate(merged):
image.order_index = idx
return merged
def _fetch_vehicle_images_with_retry(
self,
client: OpenLaneClient,
vehicle_id: str,
) -> list[dict[str, Any]]:
retry_schedule = self.openlane_config.retry_schedule_seconds
for attempt in range(len(retry_schedule) + 1):
try:
return client.fetch_vehicle_images(vehicle_id)
except OpenLaneRequestError as exc:
if exc.status_code in {401, 403}:
raise
is_retryable = exc.status_code == 429 or (
exc.status_code is not None and exc.status_code >= 500
)
if not is_retryable or attempt >= len(retry_schedule):
raise
delay = retry_schedule[attempt]
if exc.status_code == 429:
delay = max(delay * 2, 8.0)
logger.warning(
"Retry vehicle images vehicle_id=%s status=%s delay=%.1fs attempt=%d/%d",
vehicle_id,
exc.status_code,
delay,
attempt + 1,
len(retry_schedule),
)
time.sleep(delay)
return []
def _enrich_car_records_with_images(
self,
authenticated_page,
raw_records: list[dict[str, Any]],
car_records: list[CarRecord],
) -> int:
if not car_records:
return 0
raw_by_origin: dict[str, dict[str, Any]] = {}
for raw in raw_records:
raw_id = (
raw.get("id")
or raw.get("vehicle_id")
or raw.get("listing_id")
or raw.get("vin")
)
origin_id = self._normalize_origin_id(raw_id)
if origin_id:
raw_by_origin[origin_id] = raw
client = OpenLaneClient(authenticated_page, self.openlane_config)
images_added = 0
cars_by_vehicle_id: dict[str, CarRecord] = {}
for car in car_records:
raw = raw_by_origin.get(car.origin_id, {})
vehicle_id = raw.get("id") or raw.get("vehicle_id") or raw.get("listing_id")
if not vehicle_id:
# fallback: пробуем id из origin_id.
vehicle_id = car.origin_id.split(":", 1)[1] if ":" in car.origin_id else None
if not vehicle_id:
continue
cars_by_vehicle_id[str(vehicle_id)] = car
if not cars_by_vehicle_id:
return 0
vehicle_ids = list(cars_by_vehicle_id.keys())
try:
images_map = self._fetch_vehicle_images_batch_with_retry(client, vehicle_ids)
except OpenLaneRequestError as exc:
if exc.status_code in {401, 403}:
raise
logger.warning("Vehicle images batch fetch failed: %s", exc)
return 0
except Exception as exc:
logger.warning("Vehicle images batch fetch failed: %s", exc)
return 0
for vehicle_id, car in cars_by_vehicle_id.items():
images_payload = images_map.get(vehicle_id, [])
fetched_records = self._build_image_records(images_payload)
if not fetched_records:
continue
before_count = len(car.images)
# Source of truth: /api/vehicles/{id}/images.
car.images = fetched_records
after_count = len(car.images)
if after_count > before_count:
images_added += after_count - before_count
return images_added
def _fetch_vehicle_images_batch_with_retry(
self,
client: OpenLaneClient,
vehicle_ids: list[str],
) -> dict[str, list[dict[str, Any]]]:
retry_schedule = self.openlane_config.retry_schedule_seconds
for attempt in range(len(retry_schedule) + 1):
try:
return client.fetch_vehicle_images_batch(vehicle_ids)
except OpenLaneRequestError as exc:
if exc.status_code in {401, 403}:
raise
is_retryable = exc.status_code == 429 or (
exc.status_code is not None and exc.status_code >= 500
)
if not is_retryable or attempt >= len(retry_schedule):
raise
delay = retry_schedule[attempt]
if exc.status_code == 429:
delay = max(delay * 2, 8.0)
logger.warning(
"Retry vehicle images batch size=%d status=%s delay=%.1fs attempt=%d/%d",
len(vehicle_ids),
exc.status_code,
delay,
attempt + 1,
len(retry_schedule),
)
time.sleep(delay)
return {}
def init_db(self) -> dict[str, Any]:
self.persistence.create_tables()
return {"status": "ok", "message": "DB tables created"}
def sync_listing(
self,
*,
lane: str | None = None,
limit: int | None = None,
only_new: bool | None = None,
max_pages: int | None = None,
concurrency: int | None = None,
) -> dict[str, Any]:
"""Полный цикл синхронизации: OpenLane API → фильтры → маппинг → DB.
Параметры берутся из аргументов → runtime_config.json → config.py (в этом порядке).
"""
started_at = time.perf_counter()
rc = self.runtime_config
# Параметры: аргумент → runtime_config → config default.
lane = lane or rc.sync.lane or "openlane_marketplace"
if limit is None:
limit = rc.sync.limit
if limit is None and self.openlane_config.max_cars_limit > 0:
limit = self.openlane_config.max_cars_limit
if only_new is None:
only_new = rc.sync.only_new
effective_only_new = only_new if only_new is not None else False
max_pages = max_pages or self.openlane_config.max_pages
concurrency = max(1, min(concurrency or self.openlane_config.concurrency, 5))
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane)
total_upserted = 0
total_failed = 0
total_images = 0
total_discovered = 0
total_skipped = 0
all_origin_ids: set[str] = set()
completed_pages: list[int] = []
failed_pages: list[int] = []
status = "success"
error_summary: str | None = None
# Предзагружаем известные origin_id для раннего пропуска.
known_ids: set[str] = set()
if effective_only_new:
known_ids = self.persistence.get_all_origin_ids_for_lane(prefix="openlane:")
logger.info(
"sync_listing started: lane=%s max_pages=%s concurrency=%s only_new=%s known=%d",
lane, max_pages, concurrency, effective_only_new, len(known_ids),
)
try:
with sync_playwright() as playwright:
browser = self.browser_factory.create_browser(playwright)
try:
contexts = []
auth_pages = []
for i in range(concurrency):
storage_state_path = (
self.openlane_config.storage_state_file
if Path(self.openlane_config.storage_state_file).exists()
else None
)
ctx = self.browser_factory.create_context(browser, storage_state_path=storage_state_path)
auth_page = self.authenticator.bootstrap_authenticated_context(ctx)
contexts.append(ctx)
auth_pages.append(auth_page)
logger.info("Worker session initialized: worker=%d", i + 1)
# Последовательная обработка страниц (по одной на контекст).
# Для каждой страницы: fetch → map → upsert.
context_idx = 0
consecutive_all_known = 0
consecutive_failures = 0
early_stop_threshold = 3 # Остановка если 3 страницы подряд без новых записей.
failure_circuit_breaker = 2 # Остановка если 2 подряд ошибки API.
for page_num in range(1, max_pages + 1):
if limit is not None and total_upserted >= limit:
logger.info("Reached limit=%d, stopping", limit)
break
if consecutive_failures >= failure_circuit_breaker:
logger.error(
"Circuit breaker: %d consecutive failures — stopping to protect account",
consecutive_failures,
)
status = "aborted"
break
ctx = auth_pages[context_idx % len(auth_pages)]
context_idx += 1
try:
page_result = self._fetch_page_with_retry(ctx, page_num)
consecutive_failures = 0 # Сброс при успехе.
except Exception as exc:
logger.error("Page %d fetch failed: %s", page_num, exc)
failed_pages.append(page_num)
total_failed += 1
consecutive_failures += 1
continue
if not page_result.records:
logger.info("Page %d returned 0 records — end of listing", page_num)
completed_pages.append(page_num)
break
# Применяем runtime-фильтры к сырым записям.
filtered_records = apply_filters(page_result.records, rc.filters)
if len(filtered_records) < len(page_result.records):
logger.debug(
"Page %d: %d/%d records passed runtime filters",
page_num, len(filtered_records), len(page_result.records),
)
# Маппинг JSON → CarRecord.
car_records = map_openlane_records(filtered_records)
total_discovered += len(page_result.records)
if not car_records:
logger.warning("Page %d: all %d records failed mapping", page_num, len(page_result.records))
completed_pages.append(page_num)
continue
# Фильтрация уже известных записей.
if effective_only_new and known_ids:
new_records = [r for r in car_records if r.origin_id not in known_ids]
skipped = len(car_records) - len(new_records)
total_skipped += skipped
if skipped:
logger.debug("Page %d: skipped %d already known", page_num, skipped)
if not new_records:
consecutive_all_known += 1
completed_pages.append(page_num)
if consecutive_all_known >= early_stop_threshold:
logger.info(
"Early stop: %d consecutive pages with all known records",
consecutive_all_known,
)
break
continue
else:
consecutive_all_known = 0
car_records = new_records
# Применяем limit.
if limit is not None:
remaining = limit - total_upserted
car_records = car_records[:remaining]
# Дозапрашиваем фото для машин, у которых их нет в search payload.
try:
added_images = self._enrich_car_records_with_images(ctx, filtered_records, car_records)
if added_images:
logger.debug("Page %d: enriched %d images via vehicle images API", page_num, added_images)
except OpenLaneRequestError as exc:
if exc.status_code in {401, 403}:
raise
logger.warning("Page %d image enrichment skipped: %s", page_num, exc)
# Upsert батчем.
try:
upsert_result = self.persistence.upsert_cars_batch(car_records)
page_upserted = upsert_result.get("inserted", 0) + upsert_result.get("updated", 0)
page_images = upsert_result.get("images_upserted", 0)
total_upserted += page_upserted
total_images += page_images
for r in car_records:
all_origin_ids.add(r.origin_id)
known_ids.add(r.origin_id)
logger.info(
"Page %d: %d records → %d upserted, %d images",
page_num, len(car_records), page_upserted, page_images,
)
except Exception as exc:
logger.error("Page %d upsert failed: %s", page_num, exc, exc_info=True)
total_failed += len(car_records)
failed_pages.append(page_num)
continue
completed_pages.append(page_num)
finally:
browser.close()
except Exception as exc:
status = "failed"
error_summary = str(exc)[:500]
logger.error("sync_listing failed: %s", exc, exc_info=True)
elapsed = time.perf_counter() - started_at
self.persistence.finish_sync_run(
run_id,
status=status,
ids_fetched=total_discovered,
cars_upserted=total_upserted,
cars_failed=total_failed,
images_upserted=total_images,
error_summary=error_summary,
)
result = {
"run_id": run_id,
"status": status,
"lane": lane,
"total_discovered": total_discovered,
"cars_upserted": total_upserted,
"cars_failed": total_failed,
"images_upserted": total_images,
"skipped_existing": total_skipped,
"completed_pages": len(completed_pages),
"failed_pages": len(failed_pages),
"elapsed_seconds": round(elapsed, 2),
"full_scan_completed": status == "success",
}
logger.info("sync_listing finished: %s", result)
return result
def _fetch_page_with_retry(self, authenticated_page, page: int) -> OpenLanePageResult:
client = OpenLaneClient(authenticated_page, self.openlane_config)
retry_schedule = self.openlane_config.retry_schedule_seconds
for attempt in range(len(retry_schedule) + 1):
try:
return client.fetch_page(page)
except OpenLaneRequestError as exc:
# 403 — возможная блокировка, стоп.
if exc.status_code == 403:
logger.error(
"STOP: page=%d returned 403 Forbidden — aborting to protect account",
page,
)
raise
# 401 — сессия истекла.
if exc.status_code == 401:
raise
# 429 — rate limit, ретрай с увеличенным backoff.
is_retryable = exc.status_code == 429 or (
exc.status_code is not None and exc.status_code >= 500
)
if not is_retryable or attempt >= len(retry_schedule):
raise
delay = retry_schedule[attempt]
if exc.status_code == 429:
delay = max(delay * 3, 15.0) # 429 → утроенная задержка
logger.warning(
"Retry page=%d status=%s delay=%.1fs attempt=%d/%d",
page, exc.status_code, delay, attempt + 1, len(retry_schedule),
)
time.sleep(delay)
except Exception:
# Неожиданная ошибка — одна попытка.
if attempt >= min(1, len(retry_schedule)):
raise
delay = retry_schedule[attempt] if attempt < len(retry_schedule) else 5.0
logger.warning(
"Transient error page=%d delay=%.1fs attempt=%d",
page, delay, attempt + 1,
exc_info=True,
)
time.sleep(delay)
raise RuntimeError(f"Page {page} exhausted retries")

0
iaai_scraper/parsing/__init__.py → openlane_scraper/storage/__init__.py
View File

16
iaai_scraper/storage/db.py → openlane_scraper/storage/db.py
View File

@@ -11,7 +11,7 @@ from ..core.config import Settings
from .models import Base, Car, Image, SyncRun
from .schemas import CarRecord
logger = logging.getLogger("iaai_scraper.db")
logger = logging.getLogger("openlane_scraper.db")
CAR_DB_FIELDS = {
@@ -487,7 +487,7 @@ class PersistenceService:
logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "openlane") -> int:
"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
if not active_origin_ids:
return 0
@@ -496,7 +496,7 @@ class PersistenceService:
update(Car)
.where(Car.origin_id.notin_(active_origin_ids))
.where(Car.is_sold == False) # noqa: E712
.where(Car.origin_id.like("iaai:%"))
.where(Car.origin_id.like("openlane:%"))
.values(is_sold=True)
)
result = session.execute(stmt)
@@ -505,7 +505,7 @@ class PersistenceService:
logger.info("Marked %d cars as sold (no longer in listing)", count)
return count
def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "openlane") -> int:
"""Помечает авто как проданные, если их URL нет в активном листинге.
Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
@@ -543,7 +543,7 @@ class PersistenceService:
LEFT JOIN _active_urls a ON c.origin_url = a.url
WHERE a.url IS NULL
AND c.is_sold = FALSE
AND c.origin_id LIKE 'iaai:%%'
AND c.origin_id LIKE 'openlane:%%'
) sub
WHERE cars.id = sub.id
"""))
@@ -554,8 +554,8 @@ class PersistenceService:
update(Car)
.where(Car.origin_url.notin_(active_origin_urls))
.where(Car.is_sold == False) # noqa: E712
.where(Car.origin_id.like("iaai:%"))
.values(is_sold=True)
.where(Car.origin_id.like("openlane:%"))
.values(is_sold=True)
)
result = session.execute(stmt)
count = result.rowcount or 0
@@ -564,7 +564,7 @@ class PersistenceService:
logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
return count
def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
def get_all_origin_ids_for_lane(self, prefix: str = "openlane:") -> set[str]:
"""Возвращает все известные origin_id для заданного префикса.
Использует yield_per для потоковой загрузки при большом количестве записей.

1
iaai_scraper/storage/enums.py → openlane_scraper/storage/enums.py
View File

@@ -18,6 +18,7 @@ BODY_TYPE_ENUM_VALUES = (
)
COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
ORIGIN_ENUM_VALUES = (
"OPENLANE",
"IAAI",
"NA",
)

0
iaai_scraper/storage/models.py → openlane_scraper/storage/models.py
View File

0
iaai_scraper/storage/schemas.py → openlane_scraper/storage/schemas.py
View File

0
iaai_scraper/worker/__init__.py → openlane_scraper/worker/__init__.py
View File

14
iaai_scraper/worker/celery_app.py → openlane_scraper/worker/celery_app.py
View File

@@ -9,8 +9,8 @@ from redis import Redis
from ..core.config import settings
from ..core.logs import setup_logging
logger = logging.getLogger("iaai_scraper.worker.celery_app")
STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
logger = logging.getLogger("openlane_scraper.worker.celery_app")
STARTUP_SYNC_DISPATCH_KEY = "openlane:state:startup_sync_dispatched"
@celery_setup_logging.connect
@@ -37,7 +37,7 @@ def _result_backend() -> str:
celery_app = Celery(
"iaai_scraper",
"openlane_scraper",
broker=_broker_url(),
backend=_result_backend(),
)
@@ -79,7 +79,7 @@ celery_app.conf.update(
worker_hijack_root_logger=False,
beat_schedule={
"periodic-sync-listing": {
"task": "iaai_scraper.worker.tasks.sync_listing_task",
"task": "openlane_scraper.worker.tasks.sync_listing_task",
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
"args": (),
"kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False},
@@ -87,11 +87,11 @@ celery_app.conf.update(
}
},
task_routes={
"iaai_scraper.worker.tasks.*": {"queue": "scraping"}
"openlane_scraper.worker.tasks.*": {"queue": "scraping"}
},
)
celery_app.autodiscover_tasks(["iaai_scraper.worker"])
celery_app.autodiscover_tasks(["openlane_scraper.worker"])
@worker_ready.connect
@@ -118,7 +118,7 @@ def _on_worker_ready(**kwargs):
logger.info("Worker ready — dispatching initial sync_listing task")
celery_app.send_task(
"iaai_scraper.worker.tasks.sync_listing_task",
"openlane_scraper.worker.tasks.sync_listing_task",
kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
queue="scraping",
)

360
openlane_scraper/worker/tasks.py Normal file
View File

@@ -0,0 +1,360 @@
# Задачи Celery для синхронизации OpenLane marketplace → DB.
import logging
import random
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from threading import Event, Thread
from billiard.exceptions import SoftTimeLimitExceeded
from celery import shared_task
from redis import Redis
from ..core.config import Settings
from ..scraper import OpenLaneScraper
from ..storage.db import PersistenceService
logger = logging.getLogger("openlane_scraper.worker.tasks")
SYNC_LISTING_LOCK_KEY = "openlane:locks:sync_listing"
SYNC_FULL_SCAN_DONE_KEY = "openlane:state:sync_full_scan_done"
SYNC_LISTING_TASK_NAME = "openlane_scraper.worker.tasks.sync_listing_task"
def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0):
last_exc: Exception | None = None
for attempt in range(1, attempts + 1):
try:
return func()
except Exception as exc:
last_exc = exc
if attempt >= attempts:
break
delay = base_delay_s * (2 ** (attempt - 1))
logger.warning(
"Operation failed (attempt %d/%d): %s. Retrying in %.1fs",
attempt, attempts, exc, delay,
)
time.sleep(delay)
if last_exc is not None:
raise last_exc
def _run_browser_job(func, *args, **kwargs):
settings = Settings()
soft = settings.celery.task_soft_time_limit
hard = settings.celery.task_time_limit
wait_timeout = min(hard, soft + 120) if soft and hard else None
executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="openlane-browser")
future = executor.submit(func, *args, **kwargs)
try:
result = future.result(timeout=wait_timeout)
except SoftTimeLimitExceeded:
future.cancel()
executor.shutdown(wait=False, cancel_futures=True)
raise
except TimeoutError:
future.cancel()
executor.shutdown(wait=False, cancel_futures=True)
raise SoftTimeLimitExceeded("Browser thread did not finish within time limit")
except Exception:
executor.shutdown(wait=False, cancel_futures=True)
raise
else:
executor.shutdown(wait=True)
return result
def _sync_listing_lock_ttl_seconds() -> int:
settings = Settings()
soft = settings.celery.task_soft_time_limit
hard = settings.celery.task_time_limit
effective_hard = min(hard, soft + 120) if soft else hard
return max(effective_hard + 120, 300)
def _get_persistence() -> PersistenceService:
settings = Settings()
persistence = PersistenceService(settings)
def _ping_db() -> None:
with persistence.engine.connect() as conn:
conn.exec_driver_sql("SELECT 1")
_retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0)
return persistence
def _get_redis() -> Redis:
settings = Settings()
redis_client = Redis.from_url(
settings.redis.url,
decode_responses=True,
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
socket_timeout=settings.redis.socket_timeout_seconds,
health_check_interval=settings.redis.health_check_interval_seconds,
retry_on_timeout=True,
)
def _ping_redis() -> None:
redis_client.ping()
_retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0)
return redis_client
def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
try:
acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
if acquired:
return True
ttl = redis_client.ttl(key)
if ttl is not None and ttl < 0:
logger.warning("Detected stale lock without TTL, removing: %s", key)
redis_client.delete(key)
return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
return False
except Exception:
logger.warning("Failed to acquire lock %s", key, exc_info=True)
return False
def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None:
try:
refreshed = redis_client.eval(
"""
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
end
return 0
""",
1, key, owner_token, int(ttl_seconds),
)
return bool(refreshed)
except Exception:
logger.warning("Failed to refresh lock %s", key, exc_info=True)
return None
def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None:
try:
redis_client.eval(
"""
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('DEL', KEYS[1])
end
return 0
""",
1, key, owner_token,
)
except Exception:
logger.warning("Failed to release lock %s", key, exc_info=True)
def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool:
try:
inspector = celery_app.control.inspect(timeout=1.0)
snapshots = [
inspector.active() or {},
inspector.reserved() or {},
inspector.scheduled() or {},
]
except Exception:
logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True)
return True
for snapshot in snapshots:
for entries in snapshot.values():
for entry in entries or []:
task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
if task_name != SYNC_LISTING_TASK_NAME:
continue
entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "")
if exclude_task_id and entry_id == exclude_task_id:
continue
return True
return False
def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool:
try:
owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY)
if not owner_token:
return False
except Exception:
logger.warning("Failed to read sync listing lock before cleanup", exc_info=True)
return False
if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id):
logger.info("sync_listing lock preserved: active task still detected")
return False
try:
ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY)
redis_client.delete(SYNC_LISTING_LOCK_KEY)
logger.warning(
"Removed orphan sync_listing lock owner=%s ttl=%s after worker restart",
owner_token, ttl,
)
return True
except Exception:
logger.warning("Failed to clear orphan sync listing lock", exc_info=True)
return False
def _is_full_scan_done(redis_client: Redis) -> bool:
try:
value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY)
except Exception:
logger.warning("Failed to read full scan state", exc_info=True)
return False
return str(value or "").strip() == "1"
def _set_full_scan_done(redis_client: Redis, done: bool) -> None:
try:
redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0")
except Exception:
logger.warning("Failed to persist full scan state", exc_info=True)
def _start_lock_heartbeat(
redis_client: Redis, key: str, owner_token: str, ttl_seconds: int,
) -> tuple[Event, Thread]:
stop_event = Event()
interval_seconds = max(5.0, min(30.0, ttl_seconds / 3))
def _heartbeat() -> None:
while not stop_event.wait(interval_seconds):
refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds)
if refreshed is False:
logger.warning("Lost sync_listing lock ownership for %s", owner_token)
return
thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True)
thread.start()
return stop_event, thread
@shared_task(
name="openlane_scraper.worker.tasks.sync_listing_task",
bind=True,
max_retries=3,
default_retry_delay=120,
acks_late=True,
)
def sync_listing_task(
self,
lane: str = "openlane_marketplace",
limit: int | None = None,
only_new: bool | None = None,
max_pages: int | None = None,
concurrency: int | None = None,
):
"""Полный цикл синхронизации OpenLane marketplace → DB."""
persistence = _get_persistence()
persistence.create_tables()
task_id = self.request.id or "unknown"
owner_token = f"{task_id}:{uuid.uuid4().hex}"
redis_client = _get_redis()
lock_acquired = False
lock_ttl = _sync_listing_lock_ttl_seconds()
heartbeat_stop: Event | None = None
heartbeat_thread: Thread | None = None
lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
if not lock_acquired:
orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id)
if orphan_cleared:
lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
if not lock_acquired:
logger.info("sync_listing_task skipped: another sync is already running")
return {
"status": "skipped",
"reason": "sync_already_running",
"task_id": task_id,
}
try:
full_scan_done = _is_full_scan_done(redis_client)
effective_only_new = False if not full_scan_done else only_new
if not full_scan_done:
logger.info("Bootstrap mode: forcing full scan (only_new=False) until first complete run")
# Рандомный jitter (0–120s) перед стартом.
jitter = random.uniform(0, 120)
logger.info("Anti-pattern jitter: waiting %.0fs before starting scrape", jitter)
time.sleep(jitter)
heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl,
)
self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
def _job():
with OpenLaneScraper() as scraper:
return scraper.sync_listing(
lane=lane,
limit=limit,
only_new=effective_only_new,
max_pages=max_pages,
concurrency=concurrency,
)
result = _run_browser_job(_job)
if not full_scan_done:
if result.get("status") == "success":
_set_full_scan_done(redis_client, True)
logger.info("Bootstrap full scan completed; hourly schedule continues")
else:
_set_full_scan_done(redis_client, False)
summary = {
"task_id": task_id,
"run_id": result.get("run_id"),
"status": result.get("status", "success"),
"cars_upserted": result.get("cars_upserted", 0),
"cars_failed": result.get("cars_failed", 0),
"images_upserted": result.get("images_upserted", 0),
"skipped_existing": result.get("skipped_existing", 0),
"elapsed_seconds": result.get("elapsed_seconds"),
}
logger.info(
"sync_listing_task completed: status=%s, %d upserted, %d failed",
summary["status"], summary["cars_upserted"], summary["cars_failed"],
)
return summary
except SoftTimeLimitExceeded:
logger.warning("sync_listing_task soft timeout exceeded — partial progress already saved to DB")
return {
"status": "timed_out",
"task_id": task_id,
"reason": "soft_time_limit_exceeded",
}
except Exception as exc:
logger.error("sync_listing_task failed: %s", exc, exc_info=True)
try:
raise self.retry(exc=exc)
except self.MaxRetriesExceededError:
logger.error("sync_listing_task max retries exceeded, giving up")
return {
"status": "failed",
"task_id": task_id,
"error": str(exc),
}
finally:
if heartbeat_stop is not None:
heartbeat_stop.set()
if heartbeat_thread is not None:
heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
if lock_acquired:
_release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token)

8
pyproject.toml
View File

@@ -3,9 +3,9 @@ requires = ["setuptools>=68", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "iaai-scraper"
name = "openlane-scraper"
version = "0.1.0"
description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
description = "OpenLane scraper service with FastAPI, Celery, Playwright and PostgreSQL"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
@@ -31,13 +31,13 @@ dev = [
]
[project.scripts]
iaai = "iaai_scraper.cli:main"
openlane = "openlane_scraper.cli:main"
[tool.setuptools]
include-package-data = true
[tool.setuptools.packages.find]
include = ["iaai_scraper*"]
include = ["openlane_scraper*"]
[tool.pytest.ini_options]
addopts = "-q --disable-warnings"

2
runtime_config.json
View File

@@ -5,7 +5,7 @@
"ids_next_size": null,
"ids_max_pages": null,
"condition_check_enabled": false,
"lane": "iaai_cars",
"lane": "openlane_marketplace",
"only_new": false,
"limit": null
},

155
scripts/explore_site.py Normal file
View File

@@ -0,0 +1,155 @@
"""Open OpenLane in a real browser for manual exploration.
Logs ALL network requests/responses so we can understand auth flow and API structure.
Usage: python scripts/explore_site.py
"""
import json
import sys
from pathlib import Path
from datetime import datetime
from playwright.sync_api import sync_playwright, Page, Request, Response
LOG_DIR = Path("artifacts/explore")
LOG_DIR.mkdir(parents=True, exist_ok=True)
REQUESTS_LOG = LOG_DIR / f"requests_{datetime.now().strftime('%H%M%S')}.jsonl"
captured = []
def on_request(request: Request) -> None:
entry = {
"type": "request",
"ts": datetime.now().isoformat(),
"method": request.method,
"url": request.url,
"resource_type": request.resource_type,
"headers": dict(request.headers) if request.resource_type in ("xhr", "fetch", "document") else {},
}
# Log API and auth requests to console
url = request.url.lower()
if any(k in url for k in ("/api/", "/auth", "/token", "/session", "/sign_in", "/login", "/oauth")):
print(f"\n>>> {request.method} {request.url}")
if request.post_data:
# Don't print passwords
post = request.post_data[:500]
if "password" in post.lower():
post = "[CONTAINS PASSWORD - HIDDEN]"
print(f" body: {post}")
entry["post_data"] = post if "password" not in post.lower() else "[HIDDEN]"
captured.append(entry)
def on_response(response: Response) -> None:
url = response.url.lower()
if any(k in url for k in ("/api/", "/auth", "/token", "/session", "/sign_in", "/login", "/oauth")):
print(f"<<< {response.status} {response.url}")
entry = {
"type": "response",
"ts": datetime.now().isoformat(),
"status": response.status,
"url": response.url,
"headers": dict(response.headers),
}
# Try to capture response body for API/auth endpoints
try:
body = response.text()
if len(body) > 2000:
entry["body_preview"] = body[:2000] + "..."
else:
entry["body"] = body
# Print short preview
preview = body[:300] if len(body) > 300 else body
print(f" body: {preview}")
except Exception:
entry["body"] = "[could not read]"
captured.append(entry)
def save_log():
with open(REQUESTS_LOG, "w", encoding="utf-8") as f:
for entry in captured:
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
print(f"\n[*] Saved {len(captured)} entries to {REQUESTS_LOG}")
def main():
print("=" * 60)
print("OpenLane Site Explorer")
print("=" * 60)
print("1. Browser will open at OpenLane sign_in page")
print("2. Log in manually")
print("3. Browse around — all API/auth requests are logged")
print("4. When done, close the browser or press Ctrl+C here")
print("=" * 60)
with sync_playwright() as p:
browser = p.chromium.launch(
headless=False,
args=[
"--disable-blink-features=AutomationControlled",
"--no-default-browser-check",
],
)
context = browser.new_context(
viewport={"width": 1920, "height": 1080},
locale="en-US",
timezone_id="America/New_York",
user_agent=(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/135.0.0.0 Safari/537.36"
),
)
page = context.new_page()
page.on("request", on_request)
page.on("response", on_response)
print("\n[*] Opening https://app.openlane.com/sign_in ...")
page.goto("https://app.openlane.com/sign_in", wait_until="domcontentloaded")
print("\n[*] Browser is open. Log in and explore the site.")
print("[*] I'm watching all network requests...")
print("[*] When you're done, just close the browser window.\n")
try:
# Wait until the browser is closed by user
page.wait_for_event("close", timeout=0)
except KeyboardInterrupt:
print("\n[*] Interrupted by user")
except Exception:
pass
# Capture cookies and storage state before closing
try:
cookies = context.cookies()
storage = context.storage_state()
cookies_file = LOG_DIR / "cookies.json"
storage_file = LOG_DIR / "storage_state.json"
with open(cookies_file, "w", encoding="utf-8") as f:
json.dump(cookies, f, indent=2, ensure_ascii=False)
with open(storage_file, "w", encoding="utf-8") as f:
json.dump(storage, f, indent=2, ensure_ascii=False)
print(f"\n[*] Cookies saved to {cookies_file}")
print(f"[*] Storage state saved to {storage_file}")
# Print cookie names
print(f"\n[*] Cookies ({len(cookies)}):")
for c in cookies:
print(f" {c['name']} = {c['value'][:30]}... (domain={c['domain']}, httpOnly={c.get('httpOnly', '?')})")
except Exception as e:
print(f"[!] Could not save state: {e}")
browser.close()
save_log()
print("\n[*] Done! Check artifacts/explore/ for logs.")
if __name__ == "__main__":
main()

30
tests/test_db.py
View File

@@ -6,10 +6,10 @@ from pathlib import Path
from sqlalchemy import select
from iaai_scraper.core.config import Settings
from iaai_scraper.storage.db import PersistenceService
from iaai_scraper.storage.models import Car, Image, SyncRun
from iaai_scraper.storage.schemas import CarRecord, ImageRecord
from openlane_scraper.core.config import Settings
from openlane_scraper.storage.db import PersistenceService
from openlane_scraper.storage.models import Car, Image, SyncRun
from openlane_scraper.storage.schemas import CarRecord, ImageRecord
class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
@staticmethod
def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
return CarRecord(
parser_id=f"iaai:{origin_id}",
parser_id=f"openlane:{origin_id}",
brand="Toyota",
model="Camry",
year=2014,
price=price,
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id,
origin_url=f"https://app.openlane.com/vehicles/{origin_id}",
origin_id=f"openlane:{origin_id}",
slug=f"toyota-camry-{origin_id}",
images=[
ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
fullres_image="https://images.openlane.com/full/1.jpg",
preview_image="https://images.openlane.com/thumb/1.jpg",
order_index=0,
)
],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
second = self._record("888")
second.images = [
ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
fullres_image="https://images.openlane.com/full/2.jpg",
preview_image="https://images.openlane.com/thumb/2.jpg",
order_index=0,
)
]
@@ -90,7 +90,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
images = session.execute(select(Image)).scalars().all()
self.assertEqual(len(images), 1)
self.assertIn("imageKeys=2", images[0].fullres_image)
self.assertIn("full/2", images[0].fullres_image)
def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
first_run_id = self.persistence.start_sync_run("lane-a")
@@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
first = self._record("OLD-ID")
first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
first.origin_url = "https://app.openlane.com/vehicles/45089484"
self.persistence.upsert_car(first)
second = self._record("NEW-ID")
second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
second.origin_url = "https://app.openlane.com/vehicles/45089484"
result = self.persistence.upsert_car(second)
self.assertEqual(result["action"], "updated")
@@ -120,7 +120,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
cars = session.execute(select(Car)).scalars().all()
self.assertEqual(len(cars), 1)
self.assertEqual(cars[0].origin_id, "NEW-ID")
self.assertEqual(cars[0].origin_id, "openlane:NEW-ID")
if __name__ == "__main__":

68
tests/test_listing.py
View File

@@ -1,68 +0,0 @@
from __future__ import annotations
import unittest
from iaai_scraper.browser.pace import HumanPacer
from iaai_scraper.core.config import Settings
from iaai_scraper.browser.listing import ListingCollector
class _FakePage:
def __init__(self, counts: dict[str, int]) -> None:
self._counts = counts
self._evaluate_result = False
self._evaluate_values: list[object] = []
class _Locator:
def __init__(self, count_value: int) -> None:
self._count_value = count_value
def count(self) -> int:
return self._count_value
def locator(self, selector: str) -> "_FakePage._Locator":
return _FakePage._Locator(self._counts.get(selector, 0))
def evaluate(self, _script: str):
if self._evaluate_values:
return self._evaluate_values.pop(0)
return self._evaluate_result
class TestListingUnit(unittest.TestCase):
def test_pagination_detection_and_page_number(self) -> None:
# Есть кнопка Next → True.
self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1})))
# Нет селекторов → False.
self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
# Числовая пагинация через JS → True только если evaluate явно нашёл next.
page = _FakePage({})
page._evaluate_result = True
self.assertTrue(ListingCollector._has_next_page(page))
# Номер текущей страницы.
page2 = _FakePage({})
page2._evaluate_values = [5]
self.assertEqual(ListingCollector._get_current_page_number(page2), 5)
def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
collector = ListingCollector(Settings(), HumanPacer(Settings()))
html = """
<div>
<h4><a href=\"/VehicleDetail/45184893~US\">Car 1</a></h4>
<script>window.__data = {\"href\":\"\\/VehicleDetail\\/45171480~US\"}</script>
</div>
"""
links = collector._extract_vehicle_links_from_html(html)
self.assertEqual(
links,
[
("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
],
)
if __name__ == "__main__":
unittest.main()

198
tests/test_mapper.py Normal file
View File

@@ -0,0 +1,198 @@
from __future__ import annotations
import unittest
from openlane_scraper.openlane.mapper import (
map_openlane_record,
map_openlane_records,
_parse_mileage,
_parse_engine_volume_cc,
_generate_slug,
)
class TestMapOpenLaneRecord(unittest.TestCase):
def _sample_record(self, **overrides) -> dict:
base = {
"id": "12345",
"make": "Toyota",
"model": "Camry",
"year": 2020,
"mileage": 45000,
"price": 18500,
"color": "White",
"body_type": "sedan",
"transmission": "automatic",
"drivetrain": "fwd",
"vin": "1HGBH41JXMN109186",
"url": "https://app.openlane.com/vehicles/12345",
"images": [
{"full": "https://img.openlane.com/1_full.jpg", "thumbnail": "https://img.openlane.com/1_thumb.jpg"},
{"full": "https://img.openlane.com/2_full.jpg", "thumbnail": "https://img.openlane.com/2_thumb.jpg"},
],
}
base.update(overrides)
return base
def test_basic_mapping(self) -> None:
record = self._sample_record()
result = map_openlane_record(record)
self.assertIsNotNone(result)
self.assertEqual(result.brand, "TOYOTA")
self.assertEqual(result.model, "CAMRY")
self.assertEqual(result.year, 2020)
self.assertEqual(result.price, 18500)
self.assertEqual(result.mileage, 45000)
self.assertEqual(result.color, "white")
self.assertEqual(result.body_type, "SEDAN")
self.assertEqual(result.gearbox, "AT")
self.assertEqual(result.drive, "FWD")
self.assertEqual(result.origin, "OPENLANE")
self.assertEqual(result.origin_id, "openlane:12345")
self.assertEqual(result.origin_url, "https://app.openlane.com/vehicles/12345")
self.assertEqual(result.evaluation, "1HGBH41JXMN109186")
self.assertEqual(len(result.images), 2)
self.assertEqual(result.images[0].fullres_image, "https://img.openlane.com/1_full.jpg")
self.assertEqual(result.images[0].preview_image, "https://img.openlane.com/1_thumb.jpg")
def test_missing_id_returns_none(self) -> None:
record = {"make": "Toyota", "model": "Camry"}
self.assertIsNone(map_openlane_record(record))
def test_missing_brand_returns_none(self) -> None:
record = {"id": "123", "model": "Camry"}
self.assertIsNone(map_openlane_record(record))
def test_missing_model_returns_none(self) -> None:
record = {"id": "123", "make": "Toyota"}
self.assertIsNone(map_openlane_record(record))
def test_vin_as_id_fallback(self) -> None:
record = self._sample_record()
del record["id"]
record["vin"] = "WBAPH5C52BA012345"
result = map_openlane_record(record)
self.assertIsNotNone(result)
self.assertEqual(result.origin_id, "openlane:WBAPH5C52BA012345")
def test_existing_openlane_prefix_is_normalized(self) -> None:
record = self._sample_record(id="openlane:777")
result = map_openlane_record(record)
self.assertIsNotNone(result)
self.assertEqual(result.origin_id, "openlane:777")
def test_whitespace_id_is_normalized(self) -> None:
record = self._sample_record(id=" 888 ")
result = map_openlane_record(record)
self.assertIsNotNone(result)
self.assertEqual(result.origin_id, "openlane:888")
def test_image_url_strings(self) -> None:
record = self._sample_record(images=["https://a.com/1.jpg", "https://a.com/2.jpg"])
result = map_openlane_record(record)
self.assertEqual(len(result.images), 2)
self.assertEqual(result.images[0].fullres_image, "https://a.com/1.jpg")
def test_image_deduplication(self) -> None:
record = self._sample_record(images=[
{"full": "https://a.com/same.jpg"},
{"full": "https://a.com/same.jpg"},
{"full": "https://a.com/other.jpg"},
])
result = map_openlane_record(record)
self.assertEqual(len(result.images), 2)
def test_vehicle_images_api_format(self) -> None:
record = self._sample_record(images=[
{
"url": "https://a.com/original.jpg",
"large_resolution_url": "https://a.com/large.jpg",
"low_resolution_url": "https://a.com/medium.jpg",
}
])
result = map_openlane_record(record)
self.assertEqual(len(result.images), 1)
self.assertEqual(result.images[0].fullres_image, "https://a.com/original.jpg")
self.assertEqual(result.images[0].preview_image, "https://a.com/medium.jpg")
def test_default_url_generation(self) -> None:
record = self._sample_record()
del record["url"]
result = map_openlane_record(record)
self.assertEqual(result.origin_url, "https://app.openlane.com/vehicles/12345")
def test_nested_vehicle_fields(self) -> None:
record = {
"id": "999",
"vehicle": {
"make": "BMW",
"model": "X5",
"year": 2022,
"mileage": 10000,
},
}
result = map_openlane_record(record)
self.assertIsNotNone(result)
self.assertEqual(result.brand, "BMW")
self.assertEqual(result.model, "X5")
self.assertEqual(result.year, 2022)
def test_map_records_filters_invalid(self) -> None:
records = [
self._sample_record(id="1"),
{"no_id": True},
self._sample_record(id="2"),
]
results = map_openlane_records(records)
self.assertEqual(len(results), 2)
def test_sale_type_mapping(self) -> None:
record = self._sample_record(sale_type="buy_now")
result = map_openlane_record(record)
self.assertEqual(result.selling_type, "STOCK")
record2 = self._sample_record(sale_type="tender")
result2 = map_openlane_record(record2)
self.assertEqual(result2.selling_type, "TENDER")
class TestParseMileage(unittest.TestCase):
def test_integer(self) -> None:
self.assertEqual(_parse_mileage(45000), 45000)
def test_string_with_commas(self) -> None:
self.assertEqual(_parse_mileage("45,000 miles"), 45000)
def test_none_returns_zero(self) -> None:
self.assertEqual(_parse_mileage(None), 0)
def test_float(self) -> None:
self.assertEqual(_parse_mileage(45000.5), 45000)
class TestParseEngineVolume(unittest.TestCase):
def test_liters_float(self) -> None:
self.assertEqual(_parse_engine_volume_cc(2.5), 2500)
def test_cc_int(self) -> None:
self.assertEqual(_parse_engine_volume_cc(2500), 2500)
def test_string_liters(self) -> None:
self.assertEqual(_parse_engine_volume_cc("2.5L"), 2500)
def test_none(self) -> None:
self.assertIsNone(_parse_engine_volume_cc(None))
class TestGenerateSlug(unittest.TestCase):
def test_basic(self) -> None:
slug = _generate_slug(2020, "TOYOTA", "CAMRY", "openlane:123")
self.assertEqual(slug, "2020-toyota-camry-openlane-123")
def test_no_year(self) -> None:
slug = _generate_slug(None, "BMW", "X5", "openlane:456")
self.assertEqual(slug, "bmw-x5-openlane-456")
if __name__ == "__main__":
unittest.main()

94
tests/test_mappers.py
View File

@@ -1,94 +0,0 @@
from __future__ import annotations
import unittest
from iaai_scraper.parsing.mapper import CarMapper
class TestCarMapper(unittest.TestCase):
def setUp(self) -> None:
self.mapper = CarMapper()
def test_deduplicates_images_by_image_key(self) -> None:
urls = [
"https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
"https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
]
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(len(record.images), 2)
self.assertIn("width=845", record.images[0].fullres_image)
self.assertIn("height=633", record.images[0].fullres_image)
def test_no_damage_marker_is_not_damaged(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
vehicle_summary={"make": "Ford", "model": "Focus"},
payload_insights={
"vehicle_core": {},
"pricing": {},
"damage": {"primary": "normal wear"},
"auction": {},
"images": {},
},
)
self.assertFalse(record.is_damaged)
def test_unknown_empty_values_and_normalization(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.brand, "UNKNOWN")
self.assertEqual(record.model, "UNKNOWN")
self.assertEqual(record.drive, "NA")
self.assertEqual(record.gearbox, "NA")
# Нормализация регистра и пробелов.
record2 = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record2.drive, "FWD")
self.assertEqual(record2.gearbox, "AT")
def test_price_and_currency_parsing(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={
"vehicle_core": {},
"pricing": {"buy_now": "USD 4,500 - 5,200"},
"damage": {},
"auction": {},
"images": {},
},
)
self.assertEqual(record.price, 5200)
record2 = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={
"vehicle_core": {},
"pricing": {"buy_now": "€4.500,00"},
"damage": {},
"auction": {},
"images": {},
},
)
self.assertEqual(record2.currency, "EUR")
self.assertEqual(record2.price, 4500)
if __name__ == "__main__":
unittest.main()

129
tests/test_openlane.py Normal file
View File

@@ -0,0 +1,129 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from openlane_scraper.core.config import OpenLaneConfig
from openlane_scraper.openlane.checkpoint import OpenLaneCheckpoint, OpenLaneCheckpointStore
from openlane_scraper.openlane.client import OpenLaneClient
from openlane_scraper.openlane.writer import OpenLaneResultWriter
class TestOpenLaneCheckpointStore(unittest.TestCase):
def test_save_and_load_checkpoint(self) -> None:
with tempfile.TemporaryDirectory() as tmp_dir:
path = Path(tmp_dir) / "checkpoint.json"
store = OpenLaneCheckpointStore(path)
checkpoint = OpenLaneCheckpoint(
max_pages=512,
completed_pages=[1, 2, 3],
failed_pages=[7],
total_records=99,
last_saved_at="2026-04-20T00:00:00+00:00",
)
store.save(checkpoint)
loaded = store.load(max_pages=512)
self.assertEqual(loaded.max_pages, 512)
self.assertEqual(loaded.completed_pages, [1, 2, 3])
self.assertEqual(loaded.failed_pages, [7])
self.assertEqual(loaded.total_records, 99)
self.assertEqual(loaded.last_saved_at, "2026-04-20T00:00:00+00:00")
class TestOpenLaneWriter(unittest.TestCase):
def test_append_and_finalize_outputs(self) -> None:
with tempfile.TemporaryDirectory() as tmp_dir:
jsonl_path = Path(tmp_dir) / "openlane.jsonl"
aggregated_path = Path(tmp_dir) / "openlane_aggregated.json"
writer = OpenLaneResultWriter(jsonl_path, aggregated_path)
written = writer.append_page(1, [{"id": 1}, {"id": 2}])
self.assertEqual(written, 2)
summary = writer.finalize(max_pages=10, completed_pages=[1], failed_pages=[2])
self.assertEqual(summary["total_records"], 2)
self.assertEqual(summary["total_pages_completed"], 1)
self.assertEqual(summary["total_pages_failed"], 1)
self.assertTrue(aggregated_path.exists())
payload = json.loads(aggregated_path.read_text(encoding="utf-8"))
self.assertEqual(payload["items"][0]["page"], 1)
self.assertEqual(payload["items"][0]["record"]["id"], 1)
class TestOpenLaneConfig(unittest.TestCase):
def test_retry_schedule_defaults(self) -> None:
config = OpenLaneConfig()
self.assertGreaterEqual(len(config.retry_schedule_seconds), 1)
self.assertEqual(tuple(float(x) for x in config.retry_schedule_seconds), config.retry_schedule_seconds)
def test_storage_state_defaults(self) -> None:
config = OpenLaneConfig()
self.assertTrue(config.storage_state_file)
self.assertIn("artifacts/openlane", config.storage_state_file)
self.assertIsInstance(config.persist_storage_state, bool)
class TestOpenLaneClient(unittest.TestCase):
def test_extract_records_from_common_shapes(self) -> None:
self.assertEqual(
OpenLaneClient._extract_records({"results": [{"id": 1}, {"id": 2}]}),
[{"id": 1}, {"id": 2}],
)
self.assertEqual(
OpenLaneClient._extract_records({"data": {"items": [{"id": 3}]}}),
[{"id": 3}],
)
self.assertEqual(
OpenLaneClient._extract_records({"vehicles": [{"id": 4}]}),
[{"id": 4}],
)
self.assertEqual(OpenLaneClient._extract_records({"data": {"foo": "bar"}}), [])
def test_extract_vehicle_images_from_common_shapes(self) -> None:
payload = {
"vehicle_images": [
{"url": "https://img/1.png", "low_resolution_url": "https://img/1_small.png"},
]
}
self.assertEqual(
OpenLaneClient._extract_vehicle_images(payload),
[{"url": "https://img/1.png", "low_resolution_url": "https://img/1_small.png"}],
)
payload_nested = {
"data": {
"images": [
{"url": "https://img/2.png"},
]
}
}
self.assertEqual(
OpenLaneClient._extract_vehicle_images(payload_nested),
[{"url": "https://img/2.png"}],
)
self.assertEqual(OpenLaneClient._extract_vehicle_images({"data": {"foo": 1}}), [])
def test_extract_vehicle_images_batch_helper_normalization(self) -> None:
# Косвенно проверяем нормализацию входа под батчевый метод
ids = [" 123 ", "123", 456, "", None]
normalized = []
seen = set()
for v in ids:
s = str(v).strip()
if not s or s in seen:
continue
seen.add(s)
normalized.append(s)
self.assertEqual(normalized, ["123", "456", "None"])
if __name__ == "__main__":
unittest.main()

54
tests/test_parser.py
View File

@@ -1,54 +0,0 @@
from __future__ import annotations
import unittest
from iaai_scraper.parsing.parser import VehicleParser
class TestVehicleParserUnit(unittest.TestCase):
def setUp(self) -> None:
self.parser = VehicleParser()
def test_parse_dom_pairs_and_title(self) -> None:
dom_text = """
Stock #:
45089484
Primary Damage:
Front End
Odometer:
50,123 mi (Actual)
"""
result = self.parser._parse_dom_key_value_pairs(dom_text)
self.assertEqual(result.get("lot_number"), "45089484")
self.assertEqual(result.get("primary_damage"), "Front End")
self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
self.assertEqual(parsed["year"], "2014")
self.assertEqual(parsed["make"], "TOYOTA")
self.assertEqual(parsed["model"], "CAMRY")
def test_extract_image_urls_filters_and_deduplicates(self) -> None:
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
payloads = [{"imageUrls": [
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
]}]
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
self.assertEqual(len(urls), 1)
self.assertIn("45089484", urls[0])
def test_dom_hints_and_access_notes_detect_antibot(self) -> None:
hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
self.assertTrue(hints["has_captcha_text"])
self.assertTrue(hints["has_antibot_text"])
summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
notes = self.parser._build_access_notes(summary, [])
self.assertTrue(notes["possible_captcha"])
self.assertTrue(notes["possible_antibot"])
if __name__ == "__main__":
unittest.main()

273
tests/test_scraper.py
View File

@@ -1,273 +0,0 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock
from iaai_scraper.core.config import Settings
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.storage.schemas import CarRecord
def make_db_record(origin_id: str) -> dict[str, object]:
return CarRecord(
parser_id=f"iaai:{origin_id}",
brand="Toyota",
model="Camry",
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id,
slug=f"toyota-camry-{origin_id}",
).model_dump(mode="json")
class TestScraperSync(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper:
s = Settings()
s.log_level = "CRITICAL"
s.database.url = "sqlite://"
return IAAIScraper(s)
def test_sync_vehicle_uses_db_record(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=1)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
self.assertEqual(result["status"], "success")
self.assertIn("trace_id", result)
scraper.persistence.upsert_car.assert_called_once()
def test_only_new_routing_legacy_and_streaming(self) -> None:
# Legacy path: only_new без listing_url.
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=2)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
{"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
))
scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [
"https://www.iaai.com/VehicleDetail/111~US",
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
]
})
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
})
result = scraper.sync_listing(only_new=True)
self.assertEqual(result["skipped_existing"], 2)
self.assertEqual(result["cars_upserted"], 1)
# Streaming path: only_new + listing_url.
scraper2 = self._make_scraper()
scraper2.persistence.create_tables = MagicMock()
scraper2.persistence.start_sync_run = MagicMock(return_value=6)
scraper2.persistence.finish_sync_run = MagicMock()
scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used"))
scraper2._sync_listing_streaming = MagicMock(return_value={
"listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False},
"total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0,
"images_upserted": 20, "failures": [], "all_listing_origin_urls": set(),
})
result2 = scraper2.sync_listing(
only_new=True,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
year_min=2020, year_max=2027,
)
self.assertEqual(result2["cars_upserted"], 10)
scraper2._sync_listing_streaming.assert_called_once()
scraper2.collect_listing.assert_not_called()
def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=3)
scraper.persistence.finish_sync_run = MagicMock()
call_args_log: list[dict] = []
def _fake_sync_listing(**kwargs):
call_args_log.append(kwargs)
return {
"status": "success", "cars_upserted": 5, "cars_failed": 0,
"images_upserted": 10, "skipped_existing": 0,
"listing": {"vehicles_collected": 50}, "failures": [],
}
scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing)
segments = [
{"make": "TOYOTA", "year_min": 2020, "year_max": 2027},
{"make": "FORD", "year_min": None, "year_max": None},
{"make": "HONDA", "year_min": None, "year_max": None},
]
# Resume: пропускаем TOYOTA, начинаем сразу с FORD.
result = scraper.sync_listing_segmented(
segments=segments, start_segment=1,
)
self.assertEqual(result["segments_completed"], 2) # FORD + HONDA
self.assertEqual(result["cars_upserted"], 10)
# URL содержит бренд.
self.assertIn("FORD", call_args_log[0]["listing_url"])
self.assertIn("HONDA", call_args_log[1]["listing_url"])
def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None:
scraper = self._make_scraper()
scraper.sync_listing = MagicMock(side_effect=[
{
"status": "failed",
"full_scan_completed": False,
"cars_upserted": 0,
"cars_failed": 0,
"images_upserted": 0,
"skipped_existing": 0,
"listing": {"vehicles_collected": 0},
"failures": [{"vehicle_url": "segment", "error": "resume failed"}],
},
{
"status": "success",
"full_scan_completed": True,
"cars_upserted": 1,
"cars_failed": 0,
"images_upserted": 0,
"skipped_existing": 0,
"listing": {"vehicles_collected": 10},
"failures": [],
},
])
result = scraper.sync_listing_segmented(
segments=[
{"make": "EAGLE", "year_min": None, "year_max": None},
{"make": "FORD", "year_min": None, "year_max": None},
]
)
self.assertFalse(result["full_scan_completed"])
self.assertEqual(result["status"], "partial_success")
def test_build_segment_listing_url(self) -> None:
base = "https://www.iaai.com/Vehiclelisting/Cars"
self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
)
self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
)
self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
def test_guard_and_protection_detection(self) -> None:
with self.assertRaises(AntiBotDetectedError):
IAAIScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
"access_notes": {}, "vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US",
)
with self.assertRaises(SiteStructureChangedError):
IAAIScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
"access_notes": {"possible_captcha": False, "possible_antibot": False},
"vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US",
)
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
def test_close_resets_browser_state(self) -> None:
scraper = self._make_scraper()
http_pool = MagicMock()
scraper._http_pool = http_pool
scraper.context = MagicMock()
scraper.browser = MagicMock()
scraper.playwright = MagicMock()
scraper.close()
http_pool.clear.assert_called_once()
self.assertIsNone(scraper.context)
self.assertIsNone(scraper.browser)
def test_recover_empty_listing_page(self) -> None:
scraper = self._make_scraper()
page = MagicMock()
page_result = SimpleNamespace(
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
)
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper.listing_collector.open_cars_listing = MagicMock()
# Страница > 1: reload.
_, urls = scraper._recover_empty_listing_page(
page, page_number=5, all_raw_urls=[], seen_urls=set(),
)
page.reload.assert_called_once()
self.assertEqual(len(urls), 1)
# Страница 1: переоткрытие листинга.
page.reset_mock()
_, urls = scraper._recover_empty_listing_page(
page, page_number=1, all_raw_urls=[], seen_urls=set(),
)
scraper.listing_collector.open_cars_listing.assert_called_once()
page.reload.assert_not_called()
def test_sync_listing_always_starts_from_page_one(self) -> None:
scraper = self._make_scraper()
scraper.settings.celery.batch_size = 1
page = MagicMock()
page_result = SimpleNamespace(
page_number=1,
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
next_page_detected=False,
)
scraper._get_page_with_warmup = MagicMock(return_value=page)
# Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
scraper._reopen_listing_and_resume = MagicMock(
side_effect=AssertionError("pagination resume must not be used")
)
scraper.listing_collector.open_cars_listing = MagicMock()
scraper.listing_collector.apply_filters = MagicMock(return_value={
"make": None,
"model": None,
"year_min": None,
"year_max": None,
})
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 1,
"cars_failed": 0,
"images_upserted": 0,
"failures": [],
})
result = scraper._sync_listing_streaming(
make=None,
model=None,
lane="iaai_cars",
limit=None,
effective_only_new=False,
started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
)
scraper.listing_collector.open_cars_listing.assert_called_once()
scraper._reopen_listing_and_resume.assert_not_called()
scraper.sync_batch.assert_called_once()
self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(result["total"], 1)
if __name__ == "__main__":
unittest.main()

68
tests/test_scraper_images.py Normal file
View File

@@ -0,0 +1,68 @@
from __future__ import annotations
import unittest
from openlane_scraper.scraper import OpenLaneScraper
from openlane_scraper.storage.schemas import ImageRecord
class TestScraperImageHelpers(unittest.TestCase):
def test_build_image_records_supports_image_large_and_image(self) -> None:
payload = [
{
"image": "https://img.cdn/vehicle-medium-1.jpg",
"image_large": "https://img.cdn/vehicle-large-1.jpg",
},
{
"image": "https://img.cdn/vehicle-medium-2.jpg",
"large_resolution_url": "https://img.cdn/vehicle-large-2.jpg",
"low_resolution_url": "https://img.cdn/vehicle-low-2.jpg",
},
]
records = OpenLaneScraper._build_image_records(payload)
self.assertEqual(len(records), 2)
self.assertEqual(records[0].fullres_image, "https://img.cdn/vehicle-large-1.jpg")
self.assertEqual(records[0].preview_image, "https://img.cdn/vehicle-medium-1.jpg")
self.assertEqual(records[1].fullres_image, "https://img.cdn/vehicle-large-2.jpg")
self.assertEqual(records[1].preview_image, "https://img.cdn/vehicle-low-2.jpg")
def test_merge_image_records_keeps_unique_and_reindexes(self) -> None:
existing = [
ImageRecord(
fullres_image="https://img.cdn/1.jpg",
preview_image="https://img.cdn/1_small.jpg",
order_index=10,
),
ImageRecord(
fullres_image="https://img.cdn/2.jpg",
preview_image="https://img.cdn/2_small.jpg",
order_index=11,
),
]
fetched = [
ImageRecord(
fullres_image="https://img.cdn/2.jpg",
preview_image="https://img.cdn/2_small_new.jpg",
order_index=0,
),
ImageRecord(
fullres_image="https://img.cdn/3.jpg",
preview_image="https://img.cdn/3_small.jpg",
order_index=1,
),
]
merged = OpenLaneScraper._merge_image_records(existing, fetched)
self.assertEqual([img.fullres_image for img in merged], [
"https://img.cdn/2.jpg",
"https://img.cdn/3.jpg",
"https://img.cdn/1.jpg",
])
self.assertEqual([img.order_index for img in merged], [0, 1, 2])
if __name__ == "__main__":
unittest.main()

53
tests/test_utils.py
View File

@@ -2,8 +2,7 @@ from __future__ import annotations
import unittest
from iaai_scraper.core.utils import deep_find_key
from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
from openlane_scraper.core.utils import deep_find_key
class TestDeepFindKey(unittest.TestCase):
@@ -21,55 +20,5 @@ class TestDeepFindKey(unittest.TestCase):
self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"])
class TestParseListingSegments(unittest.TestCase):
def test_empty_and_invalid_return_empty(self) -> None:
self.assertEqual(parse_listing_segments(""), [])
self.assertEqual(parse_listing_segments(" "), [])
self.assertEqual(parse_listing_segments("invalid"), [])
def test_auto_segments_complete_coverage(self) -> None:
"""auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
segs = parse_listing_segments("auto")
# Точное число сегментов.
expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
self.assertEqual(len(segs), expected)
# Все бренды из списка присутствуют.
makes_in_segments = {s["make"] for s in segs}
for make in IAAI_DEFAULT_MAKES:
self.assertIn(make, makes_in_segments)
# Крупные бренды разбиты на 3 сегмента с годами.
toyota = [s for s in segs if s["make"] == "TOYOTA"]
self.assertEqual(len(toyota), 3)
for s in toyota:
self.assertIsNotNone(s["year_min"])
# Мелкие бренды без годов.
lexus = [s for s in segs if s["make"] == "LEXUS"]
self.assertEqual(len(lexus), 1)
self.assertIsNone(lexus[0]["year_min"])
# Годовые диапазоны покрывают 1950-2027.
years = set()
for yr_min, yr_max in _YEAR_SPLITS:
years.update(range(yr_min, yr_max + 1))
for year in range(1950, 2027):
self.assertIn(year, years)
def test_json_input_formats(self) -> None:
# Массив строк.
segs = parse_listing_segments('["toyota", "ford"]')
self.assertEqual(len(segs), 2)
self.assertEqual(segs[0]["make"], "TOYOTA")
# Массив объектов с годами.
segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]')
self.assertEqual(segs[0]["make"], "BMW")
self.assertEqual(segs[0]["year_min"], 2020)
self.assertEqual(segs[0]["year_max"], 2025)
if __name__ == "__main__":
unittest.main()

285
tests/test_worker_tasks.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
import unittest
from unittest.mock import MagicMock, patch
from iaai_scraper.worker import tasks
from openlane_scraper.worker import tasks
class TestWorkerTaskLockHelpers(unittest.TestCase):
@@ -52,8 +52,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("openlane_scraper.worker.tasks.time.sleep"), \
patch.object(tasks, "_run_browser_job", return_value={
"run_id": 7,
"status": "success",
"cars_upserted": 2,
"cars_failed": 0,
"images_upserted": 4,
@@ -71,7 +73,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
tasks.sync_listing_task.push_request(id="task-123")
try:
result = tasks.sync_listing_task.run(make="Toyota")
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
@@ -117,8 +119,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("openlane_scraper.worker.tasks.time.sleep"), \
patch.object(tasks, "_run_browser_job", return_value={
"run_id": 9,
"status": "success",
"cars_upserted": 3,
"cars_failed": 0,
"images_upserted": 5,
@@ -136,7 +140,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
tasks.sync_listing_task.push_request(id="task-456")
try:
result = tasks.sync_listing_task.run(make="Honda")
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
@@ -145,278 +149,41 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
self.assertEqual(acquire_lock.call_count, 2)
release_lock.assert_called_once()
def test_checkpoint_save_load_roundtrip(self) -> None:
storage: dict[str, str] = {}
def _fake_set(key, value, ex=None):
storage[key] = value
return True
redis_client = MagicMock()
redis_client.set.side_effect = _fake_set
redis_client.get.side_effect = lambda key: storage.get(key)
tasks._save_last_completed_segment(redis_client, 12)
self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
def test_load_checkpoint_clears_invalid_payload(self) -> None:
redis_client = MagicMock()
redis_client.get.return_value = "{not-a-number"
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
def test_load_checkpoint_empty_when_missing(self) -> None:
redis_client = MagicMock()
redis_client.get.return_value = None
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
redis_client.delete.assert_not_called()
def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
segments = [
{"make": "ACURA"},
{"make": "AUDI"},
{"make": "BMW"},
{"make": "EAGLE"},
]
def test_bootstrap_forces_full_scan(self) -> None:
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_set_full_scan_done") as set_done, \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks, "_release_lock_if_owner"), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
"1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
)
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_segmented_mock = MagicMock(return_value={
"run_id": 11, "status": "success", "full_scan_completed": True,
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
})
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-resume-seg")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
# last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
release_lock.assert_called_once()
def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
# Оставшийся чекпоинт не должен использоваться.
redis_client.get.side_effect = lambda key: (
"0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
)
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_segmented_mock = MagicMock(return_value={
"run_id": 14, "status": "success", "full_scan_completed": True,
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
})
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-792")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
clear_checkpoint.assert_called()
release_lock.assert_called_once()
def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
"99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
)
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_segmented_mock = MagicMock(return_value={
"run_id": 15, "status": "success", "full_scan_completed": True,
"cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
})
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-beyond")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
release_lock.assert_called_once()
def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
patch("openlane_scraper.worker.tasks.time.sleep"), \
patch.object(tasks, "_run_browser_job", return_value={
"run_id": 13,
"status": "partial_success",
"full_scan_completed": True,
"cars_upserted": 2,
"cars_failed": 1,
"images_upserted": 3,
"run_id": 11,
"status": "success",
"cars_upserted": 5,
"cars_failed": 0,
"images_upserted": 10,
"skipped_existing": 0,
"elapsed_seconds": 4.0,
"failures": [{"vehicle_url": "v", "error": "e"}],
}), \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
"elapsed_seconds": 3.0,
}):
persistence = MagicMock()
get_persistence.return_value = persistence
redis_client = MagicMock()
redis_client.get.return_value = None
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
tasks.sync_listing_task.push_request(id="task-791")
tasks.sync_listing_task.push_request(id="task-bootstrap")
try:
result = tasks.sync_listing_task.run(make="Toyota")
result = tasks.sync_listing_task.run(only_new=True)
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "partial_success")
# Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
# до браузерного job + после. Главное — вызов произошёл.
clear_checkpoint.assert_called()
release_lock.assert_called_once()
def test_try_set_followup_pending_deduplicates(self) -> None:
redis_client = MagicMock()
redis_client.set.side_effect = [True, False]
self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
redis_client = MagicMock()
redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
redis_client,
reason="max_retries_exceeded",
)
self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
self.assertFalse(should_enqueue)
redis_client.expire.assert_called_once_with(
tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
)
def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
redis_client = MagicMock()
redis_client.incr.return_value = 1
streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
redis_client,
reason="bootstrap_not_completed",
)
self.assertEqual(streak, 1)
self.assertTrue(should_enqueue)
def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
patch.object(tasks, "_clear_followup_pending") as clear_pending, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch.object(tasks, "_run_browser_job", return_value={
"run_id": 99,
"status": "failed",
"full_scan_completed": False,
"cars_upserted": 0,
"cars_failed": 0,
"images_upserted": 0,
"skipped_existing": 0,
"elapsed_seconds": 1.0,
"failures": [{"vehicle_url": "listing", "error": "bad resume"}],
"listing": {"vehicles_collected": 0},
}):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.return_value = None
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
tasks.sync_listing_task.push_request(id="task-900")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "failed")
bump_streak.assert_called_once()
clear_pending.assert_called()
task_app.send_task.assert_not_called()
self.assertEqual(result["status"], "success")
# Bootstrap should set full_scan_done to True on success.
set_done.assert_called_once_with(redis_client, True)
if __name__ == "__main__":
unittest.main()
unittest.main()

211
uv.lock generated
View File

@@ -77,6 +77,54 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" },
]
[[package]]
name = "brotli"
version = "1.2.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/f7/16/c92ca344d646e71a43b8bb353f0a6490d7f6e06210f8554c8f874e454285/brotli-1.2.0.tar.gz", hash = "sha256:e310f77e41941c13340a95976fe66a8a95b01e783d430eeaf7a2f87e0a57dd0a", size = 7388632, upload-time = "2025-11-05T18:39:42.86Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/7a/ef/f285668811a9e1ddb47a18cb0b437d5fc2760d537a2fe8a57875ad6f8448/brotli-1.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:15b33fe93cedc4caaff8a0bd1eb7e3dab1c61bb22a0bf5bdfdfd97cd7da79744", size = 863110, upload-time = "2025-11-05T18:38:12.978Z" },
{ url = "https://files.pythonhosted.org/packages/50/62/a3b77593587010c789a9d6eaa527c79e0848b7b860402cc64bc0bc28a86c/brotli-1.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:898be2be399c221d2671d29eed26b6b2713a02c2119168ed914e7d00ceadb56f", size = 445438, upload-time = "2025-11-05T18:38:14.208Z" },
{ url = "https://files.pythonhosted.org/packages/cd/e1/7fadd47f40ce5549dc44493877db40292277db373da5053aff181656e16e/brotli-1.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:350c8348f0e76fff0a0fd6c26755d2653863279d086d3aa2c290a6a7251135dd", size = 1534420, upload-time = "2025-11-05T18:38:15.111Z" },
{ url = "https://files.pythonhosted.org/packages/12/8b/1ed2f64054a5a008a4ccd2f271dbba7a5fb1a3067a99f5ceadedd4c1d5a7/brotli-1.2.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e1ad3fda65ae0d93fec742a128d72e145c9c7a99ee2fcd667785d99eb25a7fe", size = 1632619, upload-time = "2025-11-05T18:38:16.094Z" },
{ url = "https://files.pythonhosted.org/packages/89/5a/7071a621eb2d052d64efd5da2ef55ecdac7c3b0c6e4f9d519e9c66d987ef/brotli-1.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:40d918bce2b427a0c4ba189df7a006ac0c7277c180aee4617d99e9ccaaf59e6a", size = 1426014, upload-time = "2025-11-05T18:38:17.177Z" },
{ url = "https://files.pythonhosted.org/packages/26/6d/0971a8ea435af5156acaaccec1a505f981c9c80227633851f2810abd252a/brotli-1.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2a7f1d03727130fc875448b65b127a9ec5d06d19d0148e7554384229706f9d1b", size = 1489661, upload-time = "2025-11-05T18:38:18.41Z" },
{ url = "https://files.pythonhosted.org/packages/f3/75/c1baca8b4ec6c96a03ef8230fab2a785e35297632f402ebb1e78a1e39116/brotli-1.2.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9c79f57faa25d97900bfb119480806d783fba83cd09ee0b33c17623935b05fa3", size = 1599150, upload-time = "2025-11-05T18:38:19.792Z" },
{ url = "https://files.pythonhosted.org/packages/0d/1a/23fcfee1c324fd48a63d7ebf4bac3a4115bdb1b00e600f80f727d850b1ae/brotli-1.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:844a8ceb8483fefafc412f85c14f2aae2fb69567bf2a0de53cdb88b73e7c43ae", size = 1493505, upload-time = "2025-11-05T18:38:20.913Z" },
{ url = "https://files.pythonhosted.org/packages/36/e5/12904bbd36afeef53d45a84881a4810ae8810ad7e328a971ebbfd760a0b3/brotli-1.2.0-cp311-cp311-win32.whl", hash = "sha256:aa47441fa3026543513139cb8926a92a8e305ee9c71a6209ef7a97d91640ea03", size = 334451, upload-time = "2025-11-05T18:38:21.94Z" },
{ url = "https://files.pythonhosted.org/packages/02/8b/ecb5761b989629a4758c394b9301607a5880de61ee2ee5fe104b87149ebc/brotli-1.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:022426c9e99fd65d9475dce5c195526f04bb8be8907607e27e747893f6ee3e24", size = 369035, upload-time = "2025-11-05T18:38:22.941Z" },
{ url = "https://files.pythonhosted.org/packages/11/ee/b0a11ab2315c69bb9b45a2aaed022499c9c24a205c3a49c3513b541a7967/brotli-1.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:35d382625778834a7f3061b15423919aa03e4f5da34ac8e02c074e4b75ab4f84", size = 861543, upload-time = "2025-11-05T18:38:24.183Z" },
{ url = "https://files.pythonhosted.org/packages/e1/2f/29c1459513cd35828e25531ebfcbf3e92a5e49f560b1777a9af7203eb46e/brotli-1.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7a61c06b334bd99bc5ae84f1eeb36bfe01400264b3c352f968c6e30a10f9d08b", size = 444288, upload-time = "2025-11-05T18:38:25.139Z" },
{ url = "https://files.pythonhosted.org/packages/3d/6f/feba03130d5fceadfa3a1bb102cb14650798c848b1df2a808356f939bb16/brotli-1.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acec55bb7c90f1dfc476126f9711a8e81c9af7fb617409a9ee2953115343f08d", size = 1528071, upload-time = "2025-11-05T18:38:26.081Z" },
{ url = "https://files.pythonhosted.org/packages/2b/38/f3abb554eee089bd15471057ba85f47e53a44a462cfce265d9bf7088eb09/brotli-1.2.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:260d3692396e1895c5034f204f0db022c056f9e2ac841593a4cf9426e2a3faca", size = 1626913, upload-time = "2025-11-05T18:38:27.284Z" },
{ url = "https://files.pythonhosted.org/packages/03/a7/03aa61fbc3c5cbf99b44d158665f9b0dd3d8059be16c460208d9e385c837/brotli-1.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:072e7624b1fc4d601036ab3f4f27942ef772887e876beff0301d261210bca97f", size = 1419762, upload-time = "2025-11-05T18:38:28.295Z" },
{ url = "https://files.pythonhosted.org/packages/21/1b/0374a89ee27d152a5069c356c96b93afd1b94eae83f1e004b57eb6ce2f10/brotli-1.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adedc4a67e15327dfdd04884873c6d5a01d3e3b6f61406f99b1ed4865a2f6d28", size = 1484494, upload-time = "2025-11-05T18:38:29.29Z" },
{ url = "https://files.pythonhosted.org/packages/cf/57/69d4fe84a67aef4f524dcd075c6eee868d7850e85bf01d778a857d8dbe0a/brotli-1.2.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7a47ce5c2288702e09dc22a44d0ee6152f2c7eda97b3c8482d826a1f3cfc7da7", size = 1593302, upload-time = "2025-11-05T18:38:30.639Z" },
{ url = "https://files.pythonhosted.org/packages/d5/3b/39e13ce78a8e9a621c5df3aeb5fd181fcc8caba8c48a194cd629771f6828/brotli-1.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:af43b8711a8264bb4e7d6d9a6d004c3a2019c04c01127a868709ec29962b6036", size = 1487913, upload-time = "2025-11-05T18:38:31.618Z" },
{ url = "https://files.pythonhosted.org/packages/62/28/4d00cb9bd76a6357a66fcd54b4b6d70288385584063f4b07884c1e7286ac/brotli-1.2.0-cp312-cp312-win32.whl", hash = "sha256:e99befa0b48f3cd293dafeacdd0d191804d105d279e0b387a32054c1180f3161", size = 334362, upload-time = "2025-11-05T18:38:32.939Z" },
{ url = "https://files.pythonhosted.org/packages/1c/4e/bc1dcac9498859d5e353c9b153627a3752868a9d5f05ce8dedd81a2354ab/brotli-1.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:b35c13ce241abdd44cb8ca70683f20c0c079728a36a996297adb5334adfc1c44", size = 369115, upload-time = "2025-11-05T18:38:33.765Z" },
{ url = "https://files.pythonhosted.org/packages/6c/d4/4ad5432ac98c73096159d9ce7ffeb82d151c2ac84adcc6168e476bb54674/brotli-1.2.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9e5825ba2c9998375530504578fd4d5d1059d09621a02065d1b6bfc41a8e05ab", size = 861523, upload-time = "2025-11-05T18:38:34.67Z" },
{ url = "https://files.pythonhosted.org/packages/91/9f/9cc5bd03ee68a85dc4bc89114f7067c056a3c14b3d95f171918c088bf88d/brotli-1.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0cf8c3b8ba93d496b2fae778039e2f5ecc7cff99df84df337ca31d8f2252896c", size = 444289, upload-time = "2025-11-05T18:38:35.6Z" },
{ url = "https://files.pythonhosted.org/packages/2e/b6/fe84227c56a865d16a6614e2c4722864b380cb14b13f3e6bef441e73a85a/brotli-1.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8565e3cdc1808b1a34714b553b262c5de5fbda202285782173ec137fd13709f", size = 1528076, upload-time = "2025-11-05T18:38:36.639Z" },
{ url = "https://files.pythonhosted.org/packages/55/de/de4ae0aaca06c790371cf6e7ee93a024f6b4bb0568727da8c3de112e726c/brotli-1.2.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:26e8d3ecb0ee458a9804f47f21b74845cc823fd1bb19f02272be70774f56e2a6", size = 1626880, upload-time = "2025-11-05T18:38:37.623Z" },
{ url = "https://files.pythonhosted.org/packages/5f/16/a1b22cbea436642e071adcaf8d4b350a2ad02f5e0ad0da879a1be16188a0/brotli-1.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67a91c5187e1eec76a61625c77a6c8c785650f5b576ca732bd33ef58b0dff49c", size = 1419737, upload-time = "2025-11-05T18:38:38.729Z" },
{ url = "https://files.pythonhosted.org/packages/46/63/c968a97cbb3bdbf7f974ef5a6ab467a2879b82afbc5ffb65b8acbb744f95/brotli-1.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4ecdb3b6dc36e6d6e14d3a1bdc6c1057c8cbf80db04031d566eb6080ce283a48", size = 1484440, upload-time = "2025-11-05T18:38:39.916Z" },
{ url = "https://files.pythonhosted.org/packages/06/9d/102c67ea5c9fc171f423e8399e585dabea29b5bc79b05572891e70013cdd/brotli-1.2.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3e1b35d56856f3ed326b140d3c6d9db91740f22e14b06e840fe4bb1923439a18", size = 1593313, upload-time = "2025-11-05T18:38:41.24Z" },
{ url = "https://files.pythonhosted.org/packages/9e/4a/9526d14fa6b87bc827ba1755a8440e214ff90de03095cacd78a64abe2b7d/brotli-1.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:54a50a9dad16b32136b2241ddea9e4df159b41247b2ce6aac0b3276a66a8f1e5", size = 1487945, upload-time = "2025-11-05T18:38:42.277Z" },
{ url = "https://files.pythonhosted.org/packages/5b/e8/3fe1ffed70cbef83c5236166acaed7bb9c766509b157854c80e2f766b38c/brotli-1.2.0-cp313-cp313-win32.whl", hash = "sha256:1b1d6a4efedd53671c793be6dd760fcf2107da3a52331ad9ea429edf0902f27a", size = 334368, upload-time = "2025-11-05T18:38:43.345Z" },
{ url = "https://files.pythonhosted.org/packages/ff/91/e739587be970a113b37b821eae8097aac5a48e5f0eca438c22e4c7dd8648/brotli-1.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:b63daa43d82f0cdabf98dee215b375b4058cce72871fd07934f179885aad16e8", size = 369116, upload-time = "2025-11-05T18:38:44.609Z" },
{ url = "https://files.pythonhosted.org/packages/17/e1/298c2ddf786bb7347a1cd71d63a347a79e5712a7c0cba9e3c3458ebd976f/brotli-1.2.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:6c12dad5cd04530323e723787ff762bac749a7b256a5bece32b2243dd5c27b21", size = 863080, upload-time = "2025-11-05T18:38:45.503Z" },
{ url = "https://files.pythonhosted.org/packages/84/0c/aac98e286ba66868b2b3b50338ffbd85a35c7122e9531a73a37a29763d38/brotli-1.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3219bd9e69868e57183316ee19c84e03e8f8b5a1d1f2667e1aa8c2f91cb061ac", size = 445453, upload-time = "2025-11-05T18:38:46.433Z" },
{ url = "https://files.pythonhosted.org/packages/ec/f1/0ca1f3f99ae300372635ab3fe2f7a79fa335fee3d874fa7f9e68575e0e62/brotli-1.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:963a08f3bebd8b75ac57661045402da15991468a621f014be54e50f53a58d19e", size = 1528168, upload-time = "2025-11-05T18:38:47.371Z" },
{ url = "https://files.pythonhosted.org/packages/d6/a6/2ebfc8f766d46df8d3e65b880a2e220732395e6d7dc312c1e1244b0f074a/brotli-1.2.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9322b9f8656782414b37e6af884146869d46ab85158201d82bab9abbcb971dc7", size = 1627098, upload-time = "2025-11-05T18:38:48.385Z" },
{ url = "https://files.pythonhosted.org/packages/f3/2f/0976d5b097ff8a22163b10617f76b2557f15f0f39d6a0fe1f02b1a53e92b/brotli-1.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cf9cba6f5b78a2071ec6fb1e7bd39acf35071d90a81231d67e92d637776a6a63", size = 1419861, upload-time = "2025-11-05T18:38:49.372Z" },
{ url = "https://files.pythonhosted.org/packages/9c/97/d76df7176a2ce7616ff94c1fb72d307c9a30d2189fe877f3dd99af00ea5a/brotli-1.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7547369c4392b47d30a3467fe8c3330b4f2e0f7730e45e3103d7d636678a808b", size = 1484594, upload-time = "2025-11-05T18:38:50.655Z" },
{ url = "https://files.pythonhosted.org/packages/d3/93/14cf0b1216f43df5609f5b272050b0abd219e0b54ea80b47cef9867b45e7/brotli-1.2.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:fc1530af5c3c275b8524f2e24841cbe2599d74462455e9bae5109e9ff42e9361", size = 1593455, upload-time = "2025-11-05T18:38:51.624Z" },
{ url = "https://files.pythonhosted.org/packages/b3/73/3183c9e41ca755713bdf2cc1d0810df742c09484e2e1ddd693bee53877c1/brotli-1.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d2d085ded05278d1c7f65560aae97b3160aeb2ea2c0b3e26204856beccb60888", size = 1488164, upload-time = "2025-11-05T18:38:53.079Z" },
{ url = "https://files.pythonhosted.org/packages/64/6a/0c78d8f3a582859236482fd9fa86a65a60328a00983006bcf6d83b7b2253/brotli-1.2.0-cp314-cp314-win32.whl", hash = "sha256:832c115a020e463c2f67664560449a7bea26b0c1fdd690352addad6d0a08714d", size = 339280, upload-time = "2025-11-05T18:38:54.02Z" },
{ url = "https://files.pythonhosted.org/packages/f5/10/56978295c14794b2c12007b07f3e41ba26acda9257457d7085b0bb3bb90c/brotli-1.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e7c0af964e0b4e3412a0ebf341ea26ec767fa0b4cf81abb5e897c9338b5ad6a3", size = 375639, upload-time = "2025-11-05T18:38:55.67Z" },
]
[[package]]
name = "celery"
version = "5.6.3"
@@ -341,46 +389,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
]
[[package]]
name = "iaai-scraper"
version = "0.1.0"
source = { editable = "." }
dependencies = [
{ name = "alembic" },
{ name = "celery" },
{ name = "fastapi" },
{ name = "playwright" },
{ name = "psycopg2-binary" },
{ name = "pydantic" },
{ name = "python-dotenv" },
{ name = "redis" },
{ name = "sqlalchemy" },
{ name = "uvicorn" },
]
[package.optional-dependencies]
dev = [
{ name = "pytest" },
{ name = "pytest-cov" },
]
[package.metadata]
requires-dist = [
{ name = "alembic", specifier = ">=1.14.0" },
{ name = "celery", specifier = ">=5.4.0" },
{ name = "fastapi", specifier = ">=0.115.0" },
{ name = "playwright", specifier = ">=1.53.0" },
{ name = "psycopg2-binary", specifier = ">=2.9.9" },
{ name = "pydantic", specifier = ">=2.8.2" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
{ name = "python-dotenv", specifier = ">=1.0.1" },
{ name = "redis", specifier = ">=5.2.0" },
{ name = "sqlalchemy", specifier = ">=2.0.32" },
{ name = "uvicorn", specifier = ">=0.34.0" },
]
provides-extras = ["dev"]
[[package]]
name = "idna"
version = "3.11"
@@ -500,6 +508,120 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" },
]
[[package]]
name = "openlane-scraper"
version = "0.1.0"
source = { editable = "." }
dependencies = [
{ name = "alembic" },
{ name = "brotli" },
{ name = "celery" },
{ name = "fastapi" },
{ name = "orjson" },
{ name = "playwright" },
{ name = "psycopg2-binary" },
{ name = "pydantic" },
{ name = "python-dotenv" },
{ name = "redis" },
{ name = "sqlalchemy" },
{ name = "urllib3" },
{ name = "uvicorn" },
]
[package.optional-dependencies]
dev = [
{ name = "pytest" },
{ name = "pytest-cov" },
]
[package.metadata]
requires-dist = [
{ name = "alembic", specifier = ">=1.14.0" },
{ name = "brotli", specifier = ">=1.1.0" },
{ name = "celery", specifier = ">=5.4.0" },
{ name = "fastapi", specifier = ">=0.115.0" },
{ name = "orjson", specifier = ">=3.10.0" },
{ name = "playwright", specifier = ">=1.53.0" },
{ name = "psycopg2-binary", specifier = ">=2.9.9" },
{ name = "pydantic", specifier = ">=2.8.2" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
{ name = "python-dotenv", specifier = ">=1.0.1" },
{ name = "redis", specifier = ">=5.2.0" },
{ name = "sqlalchemy", specifier = ">=2.0.32" },
{ name = "urllib3", specifier = ">=2.0.0" },
{ name = "uvicorn", specifier = ">=0.34.0" },
]
provides-extras = ["dev"]
[[package]]
name = "orjson"
version = "3.11.8"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/9d/1b/2024d06792d0779f9dbc51531b61c24f76c75b9f4ce05e6f3377a1814cea/orjson-3.11.8.tar.gz", hash = "sha256:96163d9cdc5a202703e9ad1b9ae757d5f0ca62f4fa0cc93d1f27b0e180cc404e", size = 5603832, upload-time = "2026-03-31T16:16:27.878Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/67/41/5aa7fa3b0f4dc6b47dcafc3cea909299c37e40e9972feabc8b6a74e2730d/orjson-3.11.8-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:003646067cc48b7fcab2ae0c562491c9b5d2cbd43f1e5f16d98fd118c5522d34", size = 229229, upload-time = "2026-03-31T16:14:50.424Z" },
{ url = "https://files.pythonhosted.org/packages/0a/d7/57e7f2458e0a2c41694f39fc830030a13053a84f837a5b73423dca1f0938/orjson-3.11.8-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:ed193ce51d77a3830cad399a529cd4ef029968761f43ddc549e1bc62b40d88f8", size = 128871, upload-time = "2026-03-31T16:14:51.888Z" },
{ url = "https://files.pythonhosted.org/packages/53/4a/e0fdb9430983e6c46e0299559275025075568aad5d21dd606faee3703924/orjson-3.11.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f30491bc4f862aa15744b9738517454f1e46e56c972a2be87d70d727d5b2a8f8", size = 132104, upload-time = "2026-03-31T16:14:53.142Z" },
{ url = "https://files.pythonhosted.org/packages/08/4a/2025a60ff3f5c8522060cda46612d9b1efa653de66ed2908591d8d82f22d/orjson-3.11.8-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6eda5b8b6be91d3f26efb7dc6e5e68ee805bc5617f65a328587b35255f138bf4", size = 130483, upload-time = "2026-03-31T16:14:54.605Z" },
{ url = "https://files.pythonhosted.org/packages/2d/3c/b9cde05bdc7b2385c66014e0620627da638d3d04e4954416ab48c31196c5/orjson-3.11.8-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee8db7bfb6fe03581bbab54d7c4124a6dd6a7f4273a38f7267197890f094675f", size = 135481, upload-time = "2026-03-31T16:14:55.901Z" },
{ url = "https://files.pythonhosted.org/packages/ff/f2/a8238e7734de7cb589fed319857a8025d509c89dc52fdcc88f39c6d03d5a/orjson-3.11.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d8b5231de76c528a46b57010bbd83fb51e056aa0220a372fd5065e978406f1c", size = 146819, upload-time = "2026-03-31T16:14:57.548Z" },
{ url = "https://files.pythonhosted.org/packages/db/10/dbf1e2a3cafea673b1b4350e371877b759060d6018a998643b7040e5de48/orjson-3.11.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:58a4a208a6fbfdb7a7327b8f201c6014f189f721fd55d047cafc4157af1bc62a", size = 132846, upload-time = "2026-03-31T16:14:58.91Z" },
{ url = "https://files.pythonhosted.org/packages/f8/fc/55e667ec9c85694038fcff00573d221b085d50777368ee3d77f38668bf3c/orjson-3.11.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f8952d6d2505c003e8f0224ff7858d341fa4e33fef82b91c4ff0ef070f2393c", size = 133580, upload-time = "2026-03-31T16:15:00.519Z" },
{ url = "https://files.pythonhosted.org/packages/7e/a6/c08c589a9aad0cb46c4831d17de212a2b6901f9d976814321ff8e69e8785/orjson-3.11.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0022bb50f90da04b009ce32c512dc1885910daa7cb10b7b0cba4505b16db82a8", size = 142042, upload-time = "2026-03-31T16:15:01.906Z" },
{ url = "https://files.pythonhosted.org/packages/5c/cc/2f78ea241d52b717d2efc38878615fe80425bf2beb6e68c984dde257a766/orjson-3.11.8-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ff51f9d657d1afb6f410cb435792ce4e1fe427aab23d2fcd727a2876e21d4cb6", size = 423845, upload-time = "2026-03-31T16:15:03.703Z" },
{ url = "https://files.pythonhosted.org/packages/70/07/c17dcf05dd8045457538428a983bf1f1127928df5bf328cb24d2b7cddacb/orjson-3.11.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6dbe9a97bdb4d8d9d5367b52a7c32549bba70b2739c58ef74a6964a6d05ae054", size = 147729, upload-time = "2026-03-31T16:15:05.203Z" },
{ url = "https://files.pythonhosted.org/packages/90/6c/0fb6e8a24e682e0958d71711ae6f39110e4b9cd8cab1357e2a89cb8e1951/orjson-3.11.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a5c370674ebabe16c6ccac33ff80c62bf8a6e59439f5e9d40c1f5ab8fd2215b7", size = 136425, upload-time = "2026-03-31T16:15:07.052Z" },
{ url = "https://files.pythonhosted.org/packages/b2/35/4d3cc3a3d616035beb51b24a09bb872942dc452cf2df0c1d11ab35046d9f/orjson-3.11.8-cp311-cp311-win32.whl", hash = "sha256:0e32f7154299f42ae66f13488963269e5eccb8d588a65bc839ed986919fc9fac", size = 131870, upload-time = "2026-03-31T16:15:08.678Z" },
{ url = "https://files.pythonhosted.org/packages/13/26/9fe70f81d16b702f8c3a775e8731b50ad91d22dacd14c7599b60a0941cd1/orjson-3.11.8-cp311-cp311-win_amd64.whl", hash = "sha256:25e0c672a2e32348d2eb33057b41e754091f2835f87222e4675b796b92264f06", size = 127440, upload-time = "2026-03-31T16:15:09.994Z" },
{ url = "https://files.pythonhosted.org/packages/e8/c6/b038339f4145efd2859c1ca53097a52c0bb9cbdd24f947ebe146da1ad067/orjson-3.11.8-cp311-cp311-win_arm64.whl", hash = "sha256:9185589c1f2a944c17e26c9925dcdbc2df061cc4a145395c57f0c51f9b5dbfcd", size = 127399, upload-time = "2026-03-31T16:15:11.412Z" },
{ url = "https://files.pythonhosted.org/packages/01/f6/8d58b32ab32d9215973a1688aebd098252ee8af1766c0e4e36e7831f0295/orjson-3.11.8-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:1cd0b77e77c95758f8e1100139844e99f3ccc87e71e6fc8e1c027e55807c549f", size = 229233, upload-time = "2026-03-31T16:15:12.762Z" },
{ url = "https://files.pythonhosted.org/packages/a9/8b/2ffe35e71f6b92622e8ea4607bf33ecf7dfb51b3619dcfabfd36cbe2d0a5/orjson-3.11.8-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:6a3d159d5ffa0e3961f353c4b036540996bf8b9697ccc38261c0eac1fd3347a6", size = 128772, upload-time = "2026-03-31T16:15:14.237Z" },
{ url = "https://files.pythonhosted.org/packages/27/d2/1f8682ae50d5c6897a563cb96bc106da8c9cb5b7b6e81a52e4cc086679b9/orjson-3.11.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76070a76e9c5ae661e2d9848f216980d8d533e0f8143e6ed462807b242e3c5e8", size = 131946, upload-time = "2026-03-31T16:15:15.607Z" },
{ url = "https://files.pythonhosted.org/packages/52/4b/5500f76f0eece84226e0689cb48dcde081104c2fa6e2483d17ca13685ffb/orjson-3.11.8-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:54153d21520a71a4c82a0dbb4523e468941d549d221dc173de0f019678cf3813", size = 130368, upload-time = "2026-03-31T16:15:17.066Z" },
{ url = "https://files.pythonhosted.org/packages/da/4e/58b927e08fbe9840e6c920d9e299b051ea667463b1f39a56e668669f8508/orjson-3.11.8-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:469ac2125611b7c5741a0b3798cd9e5786cbad6345f9f400c77212be89563bec", size = 135540, upload-time = "2026-03-31T16:15:18.404Z" },
{ url = "https://files.pythonhosted.org/packages/56/7c/ba7cb871cba1bcd5cd02ee34f98d894c6cea96353ad87466e5aef2429c60/orjson-3.11.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14778ffd0f6896aa613951a7fbf4690229aa7a543cb2bfbe9f358e08aafa9546", size = 146877, upload-time = "2026-03-31T16:15:19.833Z" },
{ url = "https://files.pythonhosted.org/packages/0b/5d/eb9c25fc1386696c6a342cd361c306452c75e0b55e86ad602dd4827a7fd7/orjson-3.11.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea56a955056a6d6c550cf18b3348656a9d9a4f02e2d0c02cabf3c73f1055d506", size = 132837, upload-time = "2026-03-31T16:15:21.282Z" },
{ url = "https://files.pythonhosted.org/packages/37/87/5ddeb7fc1fbd9004aeccab08426f34c81a5b4c25c7061281862b015fce2b/orjson-3.11.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53a0f57e59a530d18a142f4d4ba6dfc708dc5fdedce45e98ff06b44930a2a48f", size = 133624, upload-time = "2026-03-31T16:15:22.641Z" },
{ url = "https://files.pythonhosted.org/packages/22/09/90048793db94ee4b2fcec4ac8e5ddb077367637d6650be896b3494b79bb7/orjson-3.11.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b48e274f8824567d74e2158199e269597edf00823a1b12b63d48462bbf5123e", size = 141904, upload-time = "2026-03-31T16:15:24.435Z" },
{ url = "https://files.pythonhosted.org/packages/c0/cf/eb284847487821a5d415e54149a6449ba9bfc5872ce63ab7be41b8ec401c/orjson-3.11.8-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:3f262401086a3960586af06c054609365e98407151f5ea24a62893a40d80dbbb", size = 423742, upload-time = "2026-03-31T16:15:26.155Z" },
{ url = "https://files.pythonhosted.org/packages/44/09/e12423d327071c851c13e76936f144a96adacfc037394dec35ac3fc8d1e8/orjson-3.11.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8e8c6218b614badf8e229b697865df4301afa74b791b6c9ade01d19a9953a942", size = 147806, upload-time = "2026-03-31T16:15:27.909Z" },
{ url = "https://files.pythonhosted.org/packages/b3/6d/37c2589ba864e582ffe7611643314785c6afb1f83c701654ef05daa8fcc7/orjson-3.11.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:093d489fa039ddade2db541097dbb484999fcc65fc2b0ff9819141e2ab364f25", size = 136485, upload-time = "2026-03-31T16:15:29.749Z" },
{ url = "https://files.pythonhosted.org/packages/be/c9/135194a02ab76b04ed9a10f68624b7ebd238bbe55548878b11ff15a0f352/orjson-3.11.8-cp312-cp312-win32.whl", hash = "sha256:e0950ed1bcb9893f4293fd5c5a7ee10934fbf82c4101c70be360db23ce24b7d2", size = 131966, upload-time = "2026-03-31T16:15:31.687Z" },
{ url = "https://files.pythonhosted.org/packages/ed/9a/9796f8fbe3cf30ce9cb696748dbb535e5c87be4bf4fe2e9ca498ef1fa8cf/orjson-3.11.8-cp312-cp312-win_amd64.whl", hash = "sha256:3cf17c141617b88ced4536b2135c552490f07799f6ad565948ea07bef0dcb9a6", size = 127441, upload-time = "2026-03-31T16:15:33.333Z" },
{ url = "https://files.pythonhosted.org/packages/cc/47/5aaf54524a7a4a0dd09dd778f3fa65dd2108290615b652e23d944152bc8e/orjson-3.11.8-cp312-cp312-win_arm64.whl", hash = "sha256:48854463b0572cc87dac7d981aa72ed8bf6deedc0511853dc76b8bbd5482d36d", size = 127364, upload-time = "2026-03-31T16:15:34.748Z" },
{ url = "https://files.pythonhosted.org/packages/66/7f/95fba509bb2305fab0073558f1e8c3a2ec4b2afe58ed9fcb7d3b8beafe94/orjson-3.11.8-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:3f23426851d98478c8970da5991f84784a76682213cd50eb73a1da56b95239dc", size = 229180, upload-time = "2026-03-31T16:15:36.426Z" },
{ url = "https://files.pythonhosted.org/packages/f6/9d/b237215c743ca073697d759b5503abd2cb8a0d7b9c9e21f524bcf176ab66/orjson-3.11.8-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ebaed4cef74a045b83e23537b52ef19a367c7e3f536751e355a2a394f8648559", size = 128754, upload-time = "2026-03-31T16:15:38.049Z" },
{ url = "https://files.pythonhosted.org/packages/42/3d/27d65b6d11e63f133781425f132807aef793ed25075fec686fc8e46dd528/orjson-3.11.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97c8f5d3b62380b70c36ffacb2a356b7c6becec86099b177f73851ba095ef623", size = 131877, upload-time = "2026-03-31T16:15:39.484Z" },
{ url = "https://files.pythonhosted.org/packages/dd/cc/faee30cd8f00421999e40ef0eba7332e3a625ce91a58200a2f52c7fef235/orjson-3.11.8-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:436c4922968a619fb7fef1ccd4b8b3a76c13b67d607073914d675026e911a65c", size = 130361, upload-time = "2026-03-31T16:15:41.274Z" },
{ url = "https://files.pythonhosted.org/packages/5c/bb/a6c55896197f97b6d4b4e7c7fd77e7235517c34f5d6ad5aadd43c54c6d7c/orjson-3.11.8-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ab359aff0436d80bfe8a23b46b5fea69f1e18aaf1760a709b4787f1318b317f", size = 135521, upload-time = "2026-03-31T16:15:42.758Z" },
{ url = "https://files.pythonhosted.org/packages/9c/7c/ca3a3525aa32ff636ebb1778e77e3587b016ab2edb1b618b36ba96f8f2c0/orjson-3.11.8-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f89b6d0b3a8d81e1929d3ab3d92bbc225688bd80a770c49432543928fe09ac55", size = 146862, upload-time = "2026-03-31T16:15:44.341Z" },
{ url = "https://files.pythonhosted.org/packages/3c/0c/18a9d7f18b5edd37344d1fd5be17e94dc652c67826ab749c6e5948a78112/orjson-3.11.8-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29c009e7a2ca9ad0ed1376ce20dd692146a5d9fe4310848904b6b4fee5c5c137", size = 132847, upload-time = "2026-03-31T16:15:46.368Z" },
{ url = "https://files.pythonhosted.org/packages/23/91/7e722f352ad67ca573cee44de2a58fb810d0f4eb4e33276c6a557979fd8a/orjson-3.11.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:705b895b781b3e395c067129d8551655642dfe9437273211d5404e87ac752b53", size = 133637, upload-time = "2026-03-31T16:15:48.123Z" },
{ url = "https://files.pythonhosted.org/packages/af/04/32845ce13ac5bd1046ddb02ac9432ba856cc35f6d74dde95864fe0ad5523/orjson-3.11.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:88006eda83858a9fdf73985ce3804e885c2befb2f506c9a3723cdeb5a2880e3e", size = 141906, upload-time = "2026-03-31T16:15:49.626Z" },
{ url = "https://files.pythonhosted.org/packages/02/5e/c551387ddf2d7106d9039369862245c85738b828844d13b99ccb8d61fd06/orjson-3.11.8-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:55120759e61309af7fcf9e961c6f6af3dde5921cdb3ee863ef63fd9db126cae6", size = 423722, upload-time = "2026-03-31T16:15:51.176Z" },
{ url = "https://files.pythonhosted.org/packages/00/a3/ecfe62434096f8a794d4976728cb59bcfc4a643977f21c2040545d37eb4c/orjson-3.11.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:98bdc6cb889d19bed01de46e67574a2eab61f5cc6b768ed50e8ac68e9d6ffab6", size = 147801, upload-time = "2026-03-31T16:15:52.939Z" },
{ url = "https://files.pythonhosted.org/packages/18/6d/0dce10b9f6643fdc59d99333871a38fa5a769d8e2fc34a18e5d2bfdee900/orjson-3.11.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:708c95f925a43ab9f34625e45dcdadf09ec8a6e7b664a938f2f8d5650f6c090b", size = 136460, upload-time = "2026-03-31T16:15:54.431Z" },
{ url = "https://files.pythonhosted.org/packages/01/d6/6dde4f31842d87099238f1f07b459d24edc1a774d20687187443ab044191/orjson-3.11.8-cp313-cp313-win32.whl", hash = "sha256:01c4e5a6695dc09098f2e6468a251bc4671c50922d4d745aff1a0a33a0cf5b8d", size = 131956, upload-time = "2026-03-31T16:15:56.081Z" },
{ url = "https://files.pythonhosted.org/packages/c1/f9/4e494a56e013db957fb77186b818b916d4695b8fa2aa612364974160e91b/orjson-3.11.8-cp313-cp313-win_amd64.whl", hash = "sha256:c154a35dd1330707450bb4d4e7dd1f17fa6f42267a40c1e8a1daa5e13719b4b8", size = 127410, upload-time = "2026-03-31T16:15:57.54Z" },
{ url = "https://files.pythonhosted.org/packages/57/7f/803203d00d6edb6e9e7eef421d4e1adbb5ea973e40b3533f3cfd9aeb374e/orjson-3.11.8-cp313-cp313-win_arm64.whl", hash = "sha256:4861bde57f4d253ab041e374f44023460e60e71efaa121f3c5f0ed457c3a701e", size = 127338, upload-time = "2026-03-31T16:15:59.106Z" },
{ url = "https://files.pythonhosted.org/packages/6d/35/b01910c3d6b85dc882442afe5060cbf719c7d1fc85749294beda23d17873/orjson-3.11.8-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ec795530a73c269a55130498842aaa762e4a939f6ce481a7e986eeaa790e9da4", size = 229171, upload-time = "2026-03-31T16:16:00.651Z" },
{ url = "https://files.pythonhosted.org/packages/c2/56/c9ec97bd11240abef39b9e5d99a15462809c45f677420fd148a6c5e6295e/orjson-3.11.8-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:c492a0e011c0f9066e9ceaa896fbc5b068c54d365fea5f3444b697ee01bc8625", size = 128746, upload-time = "2026-03-31T16:16:02.673Z" },
{ url = "https://files.pythonhosted.org/packages/3b/e4/66d4f30a90de45e2f0cbd9623588e8ae71eef7679dbe2ae954ed6d66a41f/orjson-3.11.8-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:883206d55b1bd5f5679ad5e6ddd3d1a5e3cac5190482927fdb8c78fb699193b5", size = 131867, upload-time = "2026-03-31T16:16:04.342Z" },
{ url = "https://files.pythonhosted.org/packages/19/30/2a645fc9286b928675e43fa2a3a16fb7b6764aa78cc719dc82141e00f30b/orjson-3.11.8-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5774c1fdcc98b2259800b683b19599c133baeb11d60033e2095fd9d4667b82db", size = 124664, upload-time = "2026-03-31T16:16:05.837Z" },
{ url = "https://files.pythonhosted.org/packages/db/44/77b9a86d84a28d52ba3316d77737f6514e17118119ade3f91b639e859029/orjson-3.11.8-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8ac7381c83dd3d4a6347e6635950aa448f54e7b8406a27c7ecb4a37e9f1ae08b", size = 129701, upload-time = "2026-03-31T16:16:07.407Z" },
{ url = "https://files.pythonhosted.org/packages/b3/ea/eff3d9bfe47e9bc6969c9181c58d9f71237f923f9c86a2d2f490cd898c82/orjson-3.11.8-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14439063aebcb92401c11afc68ee4e407258d2752e62d748b6942dad20d2a70d", size = 141202, upload-time = "2026-03-31T16:16:09.48Z" },
{ url = "https://files.pythonhosted.org/packages/52/c8/90d4b4c60c84d62068d0cf9e4d8f0a4e05e76971d133ac0c60d818d4db20/orjson-3.11.8-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa72e71977bff96567b0f500fc5bfd2fdf915f34052c782a4c6ebbdaa97aa858", size = 127194, upload-time = "2026-03-31T16:16:11.02Z" },
{ url = "https://files.pythonhosted.org/packages/8d/c7/ea9e08d1f0ba981adffb629811148b44774d935171e7b3d780ae43c4c254/orjson-3.11.8-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7679bc2f01bb0d219758f1a5f87bb7c8a81c0a186824a393b366876b4948e14f", size = 133639, upload-time = "2026-03-31T16:16:13.434Z" },
{ url = "https://files.pythonhosted.org/packages/6c/8c/ddbbfd6ba59453c8fc7fe1d0e5983895864e264c37481b2a791db635f046/orjson-3.11.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:14f7b8fcb35ef403b42fa5ecfa4ed032332a91f3dc7368fbce4184d59e1eae0d", size = 141914, upload-time = "2026-03-31T16:16:14.955Z" },
{ url = "https://files.pythonhosted.org/packages/4e/31/dbfbefec9df060d34ef4962cd0afcb6fa7a9ec65884cb78f04a7859526c3/orjson-3.11.8-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c2bdf7b2facc80b5e34f48a2d557727d5c5c57a8a450de122ae81fa26a81c1bc", size = 423800, upload-time = "2026-03-31T16:16:16.594Z" },
{ url = "https://files.pythonhosted.org/packages/87/cf/f74e9ae9803d4ab46b163494adba636c6d7ea955af5cc23b8aaa94cfd528/orjson-3.11.8-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ccd7ba1b0605813a0715171d39ec4c314cb97a9c85893c2c5c0c3a3729df38bf", size = 147837, upload-time = "2026-03-31T16:16:18.585Z" },
{ url = "https://files.pythonhosted.org/packages/64/e6/9214f017b5db85e84e68602792f742e5dc5249e963503d1b356bee611e01/orjson-3.11.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cdbc8c9c02463fef4d3c53a9ba3336d05496ec8e1f1c53326a1e4acc11f5c600", size = 136441, upload-time = "2026-03-31T16:16:20.151Z" },
{ url = "https://files.pythonhosted.org/packages/24/dd/3590348818f58f837a75fb969b04cdf187ae197e14d60b5e5a794a38b79d/orjson-3.11.8-cp314-cp314-win32.whl", hash = "sha256:0b57f67710a8cd459e4e54eb96d5f77f3624eba0c661ba19a525807e42eccade", size = 131983, upload-time = "2026-03-31T16:16:21.823Z" },
{ url = "https://files.pythonhosted.org/packages/3f/0f/b6cb692116e05d058f31ceee819c70f097fa9167c82f67fabe7516289abc/orjson-3.11.8-cp314-cp314-win_amd64.whl", hash = "sha256:735e2262363dcbe05c35e3a8869898022af78f89dde9e256924dc02e99fe69ca", size = 127396, upload-time = "2026-03-31T16:16:23.685Z" },
{ url = "https://files.pythonhosted.org/packages/c0/d1/facb5b5051fabb0ef9d26c6544d87ef19a939a9a001198655d0d891062dd/orjson-3.11.8-cp314-cp314-win_arm64.whl", hash = "sha256:6ccdea2c213cf9f3d9490cbd5d427693c870753df41e6cb375bd79bcbafc8817", size = 127330, upload-time = "2026-03-31T16:16:25.496Z" },
]
[[package]]
name = "packaging"
version = "26.0"
@@ -968,6 +1090,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" },
]
[[package]]
name = "urllib3"
version = "2.6.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" },
]
[[package]]
name = "uvicorn"
version = "0.44.0"