qananasikq/encar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: qananasikq:05b83b5518b9f55465fa1565fe61db3f0ce03e5e
Branches Tags
qananasikq:main
...
compare: qananasikq:243801f6282d80a31b83e6ce1ca6802bd86d674f
Branches Tags
qananasikq:main

10 Commits
05b83b5518 ... 243801f628

Author SHA1 Message Date
qananasikq
243801f628 update readme 2026-04-16 18:03:40 +03:00
qananasikq
80f7a1353d add tests 2026-04-16 18:01:37 +03:00
qananasikq
8140fb6f41 update alembic migrations 2026-04-16 18:01:29 +03:00
qananasikq
32824dca5f update docker config 2026-04-16 18:01:16 +03:00
qananasikq
4096d29a07 add encar scraper 2026-04-16 18:01:05 +03:00
qananasikq
5624c1973a remove iaai scraper and old tests 2026-04-16 18:00:54 +03:00
qananasikq
a4c93a1df1 fix docker scraping 
improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup
 2026-04-13 16:35:08 +03:00
qananasikq
b9557922ed fix parser_id format: car- + 22 alphanum chars, origin_id: iaai:{lot} 2026-04-10 20:31:48 +03:00
qananasikq
85b4ff8502 remove xvfb, use headless with cookie auth 2026-04-10 20:01:53 +03:00
qananasikq
6134b10fd0 add iaai auto-login with session persistence 2026-04-10 19:59:03 +03:00
68 changed files with 3776 additions and 4227 deletions
Expand all files Collapse all files

76
.env
View File

@@ -1,76 +0,0 @@
IAAI_HEADLESS=false
IAAI_LOG_LEVEL=INFO
# IAAI_LOG_FILE=iaai_scraper.log
# Network capture settings.
IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
IAAI_MAX_CAPTURED_REQUESTS=40
IAAI_MAX_CAPTURED_JSON_RESPONSES=30
# Sequential listing-first mode.
IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
IAAI_MAX_PAGES_PER_RUN=10
IAAI_MAX_VEHICLES_PER_RUN=30
IAAI_PAGE_LINK_LIMIT=100
IAAI_INCLUDE_PAGINATION=true
IAAI_COLLECT_CURRENT_PAGE_ONLY=false
# Human-like pacing.
IAAI_HUMAN_PACE_ENABLED=true
IAAI_AFTER_LISTING_OPEN_MIN_S=2.5
IAAI_AFTER_LISTING_OPEN_MAX_S=4.5
IAAI_AFTER_FILTER_ACTION_MIN_S=2.0
IAAI_AFTER_FILTER_ACTION_MAX_S=4.0
IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5
IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0
IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0
IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5
IAAI_BETWEEN_VEHICLES_MIN_S=3.0
IAAI_BETWEEN_VEHICLES_MAX_S=6.0
IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
# Sync settings
IAAI_SYNC_ONLY_NEW=true
IAAI_TOKENS_FILE=tokens.json
IAAI_RUNTIME_CONFIG_FILE=runtime_config.json
# Retry / backoff
IAAI_RETRY_DELAY_SECONDS=2.5
IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
IAAI_RETRY_JITTER_SECONDS=0.25
# Session persistence (cookies)
IAAI_STORAGE_STATE_PATH=storage_state.json
IAAI_SESSION_MAX_AGE_DAYS=30
IAAI_SESSION_SAVE_ON_EXIT=true
# Proxy (HTTP/HTTPS preferred for Playwright)
# Chromium does not support SOCKS5 proxy authentication directly.
# Use residential or mobile USA proxy.
# IAAI_PROXY_SERVER=http://proxy.example.com:8080
# IAAI_PROXY_USERNAME=
# IAAI_PROXY_PASSWORD=
# Database — SQLite (локальный запуск без Docker).
# Для PostgreSQL: postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
IAAI_DATABASE_ECHO=false
IAAI_DATABASE_POOL_SIZE=5
IAAI_DATABASE_MAX_OVERFLOW=10
IAAI_DATABASE_AUTO_CREATE_TABLES=true
# Redis (Celery broker) не нужен для CLI-режима.
# Раскомментировать для запуска API + Worker + Beat.
# IAAI_REDIS_URL=redis://localhost:6379/0
# Celery —не нужен для CLI-режима.
# CELERY_BROKER_URL=redis://localhost:6379/0
# CELERY_RESULT_BACKEND=redis://localhost:6379/0
# CELERY_TASK_SOFT_TIME_LIMIT=600
# CELERY_TASK_TIME_LIMIT=900
# CELERY_WORKER_CONCURRENCY=1
# CELERY_WORKER_MAX_TASKS_PER_CHILD=20
# CELERY_BROKER_VISIBILITY_TIMEOUT=7200
# CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
# CELERY_BEAT_SYNC_LIMIT=26

90
.env.example
View File

@@ -1,72 +1,28 @@
IAAI_HEADLESS=true
IAAI_LOG_LEVEL=INFO
# IAAI_LOG_FILE=iaai_scraper.log
# ===== Encar Scraper Config =====
# Network capture settings.
IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
IAAI_MAX_CAPTURED_REQUESTS=40
IAAI_MAX_CAPTURED_JSON_RESPONSES=30
# Логирование
ENCAR_LOG_LEVEL=INFO
# Sequential listing-first mode.
IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
IAAI_MAX_PAGES_PER_RUN=10
IAAI_MAX_VEHICLES_PER_RUN=30
IAAI_PAGE_LINK_LIMIT=100
IAAI_INCLUDE_PAGINATION=true
IAAI_COLLECT_CURRENT_PAGE_ONLY=false
# База данных PostgreSQL
ENCAR_DATABASE_URL=postgresql+psycopg2://encar:encar@postgres:5432/encar_db
ENCAR_DATABASE_ECHO=false
ENCAR_DATABASE_POOL_SIZE=5
ENCAR_DATABASE_MAX_OVERFLOW=10
ENCAR_DATABASE_POOL_RECYCLE_SECONDS=1800
# Human-like pacing.
IAAI_HUMAN_PACE_ENABLED=true
IAAI_AFTER_LISTING_OPEN_MIN_S=2.5
IAAI_AFTER_LISTING_OPEN_MAX_S=4.5
IAAI_AFTER_FILTER_ACTION_MIN_S=2.0
IAAI_AFTER_FILTER_ACTION_MAX_S=4.0
IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5
IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0
IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0
IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5
IAAI_BETWEEN_VEHICLES_MIN_S=3.0
IAAI_BETWEEN_VEHICLES_MAX_S=6.0
IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
# Redis
ENCAR_REDIS_URL=redis://redis:6379/1
# Sync settings
IAAI_SYNC_ONLY_NEW=true
IAAI_TOKENS_FILE=/data/tokens.json
IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
# Celery
CELERY_BROKER_URL=redis://redis:6379/1
CELERY_RESULT_BACKEND=redis://redis:6379/1
CELERY_TASK_SOFT_TIME_LIMIT=14400
CELERY_TASK_TIME_LIMIT=14520
CELERY_BROKER_VISIBILITY_TIMEOUT=15000
CELERY_WORKER_CONCURRENCY=4
CELERY_WORKER_MAX_TASKS_PER_CHILD=50
# Retry / backoff
IAAI_RETRY_DELAY_SECONDS=2.5
IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
IAAI_RETRY_JITTER_SECONDS=0.25
# Session persistence (cookies + localStorage)
# Keeps browser session alive between runs (up to 30 days).
IAAI_STORAGE_STATE_PATH=storage_state.json
IAAI_SESSION_MAX_AGE_DAYS=30
IAAI_SESSION_SAVE_ON_EXIT=true
# Proxy (HTTP/HTTPS preferred for Playwright)
# Chromium does not support SOCKS5 proxy authentication directly.
# Use residential or mobile USA proxy.
# IAAI_PROXY_SERVER=http://proxy.example.com:8080
# IAAI_PROXY_USERNAME=
# IAAI_PROXY_PASSWORD=
# Database (PostgreSQL).
IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
IAAI_DATABASE_ECHO=false
IAAI_DATABASE_POOL_SIZE=5
IAAI_DATABASE_MAX_OVERFLOW=10
# ─── Redis (Celery broker) ─────────────────────────────────
IAAI_REDIS_URL=redis://redis:6379/0
# ─── Celery ────────────────────────────────────────────────
CELERY_BROKER_URL=redis://redis:6379/0
CELERY_RESULT_BACKEND=redis://redis:6379/0
CELERY_TASK_SOFT_TIME_LIMIT=600
CELERY_TASK_TIME_LIMIT=900
CELERY_WORKER_CONCURRENCY=1
CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
CELERY_BEAT_SYNC_LIMIT=26
# Encar Beat — полный прогон каждые 60 минут
ENCAR_BEAT_INTERVAL_MINUTES=60
# 0 = без лимита, парсит ВСЕ авто
ENCAR_BEAT_LIMIT=0

3
.gitignore vendored
View File

@@ -16,4 +16,5 @@ dist/
build/
artifacts/
celerybeat-schedule*
tokens_data/
tokens_data/
.env

14
Dockerfile
View File

@@ -1,4 +1,4 @@
FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
@@ -7,20 +7,14 @@ RUN groupadd --system app && useradd --system --gid app --create-home app
WORKDIR /app
# Xvfb for headed Chromium in container (IAAI blocks headless)
RUN apt-get update -qq && apt-get install -y --no-install-recommends xvfb \
&& rm -rf /var/lib/apt/lists/*
# Required for non-root Xvfb runtime in containers
RUN mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix
COPY pyproject.toml uv.lock ./
RUN pip install --no-cache-dir uv \
&& uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
&& pip install --no-cache-dir -r /tmp/requirements.txt
COPY . .
RUN python -m compileall -q iaai_scraper
RUN pip install --no-cache-dir -e .
RUN python -m compileall -q encar_scraper
RUN chmod +x entrypoint.sh
RUN chown -R app:app /app
@@ -30,4 +24,4 @@ USER app
# По умолчанию API, но worker/beat переопределяют CMD в docker-compose
ENTRYPOINT ["./entrypoint.sh"]
CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
CMD ["uvicorn", "encar_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

400
README.md
View File

@@ -1,342 +1,114 @@
# IAAI Scraper
# Encar Scraper
Парсер аукционных автомобилей с [iaai.com](https://www.iaai.com). Ходит по листингу, собирает карточки машин, вытаскивает данные из DOM и перехваченных XHR-ответов, складывает всё в PostgreSQL. Работает через Playwright, FastAPI, Celery и Docker.
Парсер автомобилей с [encar.com](https://www.encar.com) — крупнейшей корейской площадки продажи авто.
## Как устроен сайт и его защита
## Как это работает
У IAAI стоит **Imperva Incapsula** — внешний WAF и anti-bot.
Скрапер работает через публичный JSON API Encar (без браузера, без Selenium):
- **Anti-bot** — headless Chromium без прокси часто режется.
- **Динамическая подгрузка** — часть данных приходит через XHR (`/Search`, `/VehicleDetail`), часть остаётся в HTML.
- **Cookie consent** — при первом заходе показывают баннер.
1. **Листинг** — `api.encar.com/search/car/list/general` отдаёт список авто с пагинацией (до 1000 на страницу)
2. **Шардирование** — API лимитирует выдачу ~10k результатов, поэтому весь каталог (~230k авто) разбивается на 36 шардов по типу (domestic/import) и диапазонам годов
3. **Фото** — batch endpoint `api.encar.com/v1/readside/vehicles` (до 20 ID за запрос) возвращает все фото с CDN `ci.encar.com`
4. **Переводы** — корейские названия брендов, моделей и цветов автоматически переводятся в русские/английские
5. **Sold-трекинг** — авто, пропавшие из листинга, помечаются как проданные
Поэтому в проекте используются Playwright, паузы между действиями, прокси и сохранение браузерного состояния.
Celery Beat запускает полную синхронизацию каждые 60 минут. Worker обходит все шарды, парсит данные и пишет батчами в PostgreSQL.
## Локальный запуск (без Docker)
## Особенности
### Требования
- **Чистые HTTP запросы** — без браузера, через публичный API Encar
- **~100 авто/сек** — полный sync ~230k авто за ~35 мин
- **PostgreSQL** для хранения данных (авто + фото)
- **Celery + Redis** для фоновых задач и периодической синхронизации
- **FastAPI** REST API для управления задачами и просмотра данных
- **Python 3.11+**
- **Git**
Docker **не нужен**. Данные хранятся в SQLite-файле `iaai_scraper.db` в корне проекта.
### Быстрый старт
## Быстрый старт (Docker)
```bash
git clone <repo-url>
cd iaai_scraper_project
pip install -e .
playwright install chromium
iaai init-db
docker compose up -d --build
```
Готовый `.env` уже в репозитории и настроен на SQLite ничего менять не нужно.
### Запуск парсера
**Скрапинг одной машины:**
```bash
iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
```
**Сбор листинга + скрапинг (например Toyota, 10 штук):**
```bash
iaai sync-listing --make Toyota --limit 10
```
**Только ссылки с листинга (без скрапинга):**
```bash
iaai collect-listing --make Toyota
```
**С видимым браузером (для отладки):**
```bash
iaai --headless false sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
```
**Проверка, что записалось в базу (`iaai_scraper.db`):**
```bash
python -c "import sqlite3; c=sqlite3.connect('iaai_scraper.db'); q=c.cursor(); print('cars:', q.execute('select count(*) from cars').fetchone()[0]); print('images:', q.execute('select count(*) from images').fetchone()[0]); rows=q.execute('select id, brand, model, year, origin_id from cars order by id desc limit 10').fetchall(); [print(r) for r in rows]"
```
Результаты сохраняются в `artifacts/json/` и в БД `iaai_scraper.db`.
### Полный стек (API + Worker + Beat)
Для API и автоматического сбора нужны **Redis** и **PostgreSQL**. В `.env` раскомментируй строки Redis/Celery и замени БД на PostgreSQL:
```env
IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
IAAI_REDIS_URL=redis://localhost:6379/0
CELERY_BROKER_URL=redis://localhost:6379/0
CELERY_RESULT_BACKEND=redis://localhost:6379/0
```
Применить миграции и запустить в трёх терминалах:
```bash
alembic upgrade head
# Терминал 1 — API
uvicorn iaai_scraper.api.app:app --reload --port 8000
# Терминал 2 — Celery Worker
celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
# Терминал 3 — Celery Beat (периодический запуск)
celery -A iaai_scraper.worker.celery_app beat --loglevel=info
```
API: `http://localhost:8000` · Swagger: `http://localhost:8000/docs`
---
## Запуск через Docker (все сервисы в контейнерах)
```bash
cp .env.example .env
docker compose up -d
```
Будут запущены сервисы `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`. API доступен на `http://localhost:8000`.
В Docker-образ дополнительно проверяется Python-синтаксис на этапе сборки (`python -m compileall -q iaai_scraper`), чтобы не выкатывать битый код.
`entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
Swagger-документация: `http://localhost:8000/docs`
```bash
docker compose ps
```
- `api` имеет healthcheck на `GET /health`
- `worker` имеет healthcheck через `celery inspect ping`
- `beat` имеет healthcheck по файлу `celerybeat-schedule`
## API
**Здоровье и статистика:**
- `GET /health` — статус сервиса и подключения к БД
- `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
**Машины:**
- `GET /api/v1/cars` — список с пагинацией
- `GET /api/v1/cars/{id}` — карточка с картинками
- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number
**Задачи:**
- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL
- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга
- `GET /api/v1/sync-runs` — история запусков
Скрапим конкретную машину:
```bash
curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle \
-H "Content-Type: application/json" \
-d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}'
```
Поднимутся сервисы:
- `encar-postgres` — PostgreSQL
- `encar-redis` — Redis
- `encar-api` — FastAPI на порту 8000
- `encar-worker` — Celery worker
- `encar-beat` — периодическая синхронизация (каждые 60 мин)
## CLI
Для отладки без API и Celery:
```bash
# Инициализация БД
encar init-db
# Собрать листинг
encar collect-listing --limit 100
# Синхронизировать листинг в БД
encar sync-listing --limit 100 --car-type all --only-new true
# Синхронизировать одно авто
encar sync-vehicle "https://www.encar.com/dc/dc_cardetailview.do?carid=41421262"
```
### Фильтры
```bash
iaai init-db
iaai collect-listing --make Toyota
iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
iaai sync-listing --limit 10
encar sync-listing --car-type import --manufacturer BMW --year-from 2020 --limit 50
```
## Структура
## API endpoints
```text
iaai_scraper/
scraper.py — оркестратор: связывает browser → parser → storage
cli.py — CLI-команды (init-db, sync-vehicle, sync-listing, ...)
proxy_bridge.py — HTTP→SOCKS5 мост (Chromium не умеет SOCKS5 с авторизацией)
- `GET /health` — проверка доступности
- `GET /api/v1/cars` — список авто с пагинацией и фильтрами
- `GET /api/v1/cars/{car_id}` — детали авто
- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по origin_id
- `POST /api/v1/tasks/sync-vehicle` — синхронизация одного авто
- `POST /api/v1/tasks/sync-listing` — синхронизация листинга
- `GET /api/v1/tasks/{task_id}` — статус задачи
- `GET /api/v1/sync-runs` — история синхронизаций
browser/
factory.py — создание браузера, stealth-инъекции, fingerprint
listing.py — сбор ссылок на машины с листинга, пагинация
network.py — перехват XHR/fetch ответов через Playwright events
pace.py — рандомные паузы и движение мыши
parsing/
parser.py — VehicleParser: DOM + XHR JSON + embedded JSON
mapper.py — CarMapper: нормализация → CarRecord
storage/
models.py — SQLAlchemy: Car, Image, SyncRun
schemas.py — Pydantic: CarRecord, ImageRecord, CarRead
enums.py — допустимые значения (drive, gearbox, body_type, ...)
db.py — PersistenceService: upsert, sync runs, статистика
api/
app.py — FastAPI factory, lifespan, роутеры
deps.py — dependency injection (Settings, PersistenceService)
routes/
health.py — GET /health
cars.py — CRUD по машинам + GET /stats
tasks.py — запуск Celery-задач и история sync-runs
worker/
celery_app.py — конфиг Celery, beat-расписание
tasks.py — sync_vehicle_task, sync_listing_task
core/
config.py — Settings (dataclass), env-переменные
logs.py — логирование с trace_id (ContextVar)
retry.py — декоратор @retryable с exponential backoff
runtime_config.py — чтение и нормализация runtime-конфига
utils.py — VIN_RE, deep_find_key, вспомогательные функции
alembic/ — миграции БД
tests/ — тесты
```
## Конфигурация
Всё через env-переменные (полный список в `.env.example`):
- **БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE`
- **Redis:** `IAAI_REDIS_URL`
- **Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
- **Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN`
- **Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD`
- **Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д.
## Миграции
В Docker миграции выполняются отдельным сервисом `migrate` (`alembic upgrade head`).
`api`, `worker`, `beat` стартуют после успешного завершения `migrate`.
Вручную:
### Примеры
```bash
alembic upgrade head
alembic revision --autogenerate -m "add_column_x"
# Запуск синхронизации
curl -X POST http://localhost:8000/api/v1/tasks/sync-listing \
-H "Content-Type: application/json" \
-d '{"car_type": "all", "limit": 100, "only_new": true}'
# Статус задачи
curl http://localhost:8000/api/v1/tasks/<task_id>
# Список авто
curl "http://localhost:8000/api/v1/cars?page=1&per_page=20"
```
## Тесты
## Переменные окружения
```bash
pytest -q
- **БД:** `ENCAR_DATABASE_URL`
- **Redis:** `ENCAR_REDIS_URL`
- **Celery:** `CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`, `CELERY_TASK_TIME_LIMIT`
- **Beat:** `ENCAR_BEAT_INTERVAL_MINUTES` (по умолчанию 60), `ENCAR_BEAT_LIMIT` (0 = без лимита)
## Структура проекта
```
encar_scraper/
├── encar.py — скрапер Encar (HTTP API), маппер, переводы
├── cli.py — CLI интерфейс
├── api/
│ ├── app.py — FastAPI приложение
│ └── routes/ — health, cars, tasks
├── core/
│ ├── config.py — настройки из env vars
│ ├── utils.py — утилиты (save_to_json, deep_find_key)
│ └── logs.py — логирование с trace_id
├── storage/
│ ├── db.py — PersistenceService (upsert, session_scope)
│ ├── models.py — SQLAlchemy модели (Car, Image, SyncRun)
│ └── schemas.py — Pydantic схемы (CarRecord, ImageRecord)
└── worker/
├── celery_app.py — Celery app + beat schedule
└── tasks.py — encar_sync_listing_task, encar_sync_vehicle_task
```
Тесты работают на SQLite in-memory, без внешних зависимостей.
## `pyproject.toml` + `uv.lock`
Проект переведён на современную схему зависимостей:
- `pyproject.toml` — декларация зависимостей и метаданных проекта
- `uv.lock` — зафиксированные версии для воспроизводимых установок
Локально можно использовать:
```bash
uv sync
uv run pytest -q
```
## Runtime-фильтры
Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
### Секция `sync`
Поддерживаются поля:
- `name`
- `ids_initial_size`
- `ids_next_size`
- `ids_max_pages`
- `condition_check_enabled`
- `lane`
- `only_new`
- `limit`
Так же используются: `lane`, `only_new`, `limit`.
### Секция `filters`
Поддерживаются поля:
- `brands`
- `models`
- `years`
- `body_types`
- `colors`
- `drives`
- `gearboxes`
- `locations`
- `exclude_brands`
- `exclude_models`
- `exclude_years`
- `exclude_body_types`
- `exclude_colors`
- `exclude_drives`
- `exclude_gearboxes`
- `exclude_locations`
- `price.min`, `price.max`
- `mileage.min`, `mileage.max`
- `flags.damaged_only`, `flags.run_and_drive`
Пример:
```json
{
"sync": {
"name": null,
"ids_initial_size": null,
"ids_next_size": null,
"ids_max_pages": null,
"condition_check_enabled": false,
"lane": "iaai_cars",
"only_new": true,
"limit": 50
},
"filters": {
"price": {
"min": null,
"max": null
},
"mileage": {
"min": null,
"max": null
},
"flags": {
"damaged_only": null,
"run_and_drive": null
},
"brands": ["Toyota", "Honda"],
"models": [],
"years": [2021, 2022],
"body_types": ["SUV"],
"colors": [],
"drives": [],
"gearboxes": [],
"locations": [],
"exclude_brands": [],
"exclude_models": [],
"exclude_years": [],
"exclude_body_types": []
}
}
```
Путь задаётся через `IAAI_RUNTIME_CONFIG_FILE`, по умолчанию — `/app/runtime_config.json`.
CLI и API аргументы имеют приоритет, а `runtime_config.json` работает как runtime-default и расширяемый фильтр.

2
alembic.ini
View File

@@ -3,7 +3,7 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
sqlalchemy.url = postgresql+psycopg2://encar:encar@localhost:5434/encar_db
[loggers]
keys = root,sqlalchemy,alembic

6
alembic/env.py
View File

@@ -1,4 +1,4 @@
# Alembic env.py — подключение к БД через Settings.
# Alembic env.py — подключение к БД через Settings.
import os
import sys
@@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool
# Добавляем корень проекта в sys.path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from iaai_scraper.core.config import Settings
from iaai_scraper.storage.models import Base
from encar_scraper.core.config import Settings
from encar_scraper.storage.models import Base
config = context.config

13
alembic/versions/001_initial.py
View File

@@ -1,9 +1,4 @@
"""Initial schema — cars, images, sync_runs
Revision ID: 001_initial
Revises:
Create Date: 2026-04-08
"""
# Начальная схема: cars, images, sync_runs
from typing import Sequence, Union
from alembic import op
@@ -16,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# --- cars ---
# Таблица cars — автомобили с Encar
op.create_table(
"cars",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
@@ -53,7 +48,7 @@ def upgrade() -> None:
op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
# --- images ---
# Таблица images — изображения машин
op.create_table(
"images",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
@@ -63,7 +58,7 @@ def upgrade() -> None:
sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
)
# --- sync_runs ---
# Таблица sync_runs — логирование синхронизаций
op.create_table(
"sync_runs",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),

20
alembic/versions/002_add_indexes.py
View File

@@ -1,9 +1,4 @@
"""Add performance indexes for growing database
Revision ID: 002_add_indexes
Revises: 001_initial
Create Date: 2026-04-09
"""
# Индексы производительности
from typing import Sequence, Union
from alembic import op
@@ -15,25 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# cars: ускорение фильтрации по бренду в API и статистике
op.create_index("ix_cars_brand", "cars", ["brand"])
# cars: составной индекс бренд+модель для комбинированных фильтров
op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
# cars: ускорение фильтрации по году (year_min/year_max)
op.create_index("ix_cars_year", "cars", ["year"])
# cars: ускорение фильтрации по статусу продажи
op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
# cars: ускорение сортировки ORDER BY last_seen_at DESC (пагинация)
op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
# images: ускорение JOIN/DELETE по car_id (критично при upsert)
op.create_index("ix_images_car_id", "images", ["car_id"])
# sync_runs: ускорение поиска stale runs по статусу
op.create_index("ix_sync_runs_status", "sync_runs", ["status"])

38
alembic/versions/003_add_composite_indexes.py Normal file
View File

@@ -0,0 +1,38 @@
# Индексы для масштабированной БД (20k+ записей)
from typing import Sequence, Union
from alembic import op
revision: str = "003_add_composite_indexes"
down_revision: Union[str, None] = "002_add_indexes"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Partial index для активных машин Encar (is_sold = FALSE)
# Ускоряет поиск при mark_sold операциях
op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_active_encar "
"ON cars (origin_url) "
"WHERE is_sold = FALSE AND origin_id LIKE 'encar:%'"
)
# Composite index для проверки дубликатов изображений
op.create_index(
"ix_images_car_id_fullres",
"images",
["car_id", "fullres_image"],
)
# Index для быстрого поиска existing машин по origin_url
op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
"ON cars (origin_url, origin_id)"
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
op.drop_index("ix_images_car_id_fullres", table_name="images")
op.execute("DROP INDEX IF EXISTS ix_cars_active_encar")

25
alembic/versions/004_widen_string_columns.py Normal file
View File

@@ -0,0 +1,25 @@
"""widen parser_id brand model columns
Revision ID: 004
Revises: 003_add_composite_indexes
Create Date: 2026-04-15
"""
from alembic import op
import sqlalchemy as sa
revision = "004_widen_string_columns"
down_revision = "003_add_composite_indexes"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.alter_column("cars", "parser_id", type_=sa.String(255), existing_nullable=False)
op.alter_column("cars", "brand", type_=sa.String(100), existing_nullable=False)
op.alter_column("cars", "model", type_=sa.String(500), existing_nullable=False)
def downgrade() -> None:
op.alter_column("cars", "parser_id", type_=sa.String(50), existing_nullable=False)
op.alter_column("cars", "brand", type_=sa.String(50), existing_nullable=False)
op.alter_column("cars", "model", type_=sa.String(50), existing_nullable=False)

33
alembic/versions/005_unique_images_constraint.py Normal file
View File

@@ -0,0 +1,33 @@
"""add unique constraint on images(car_id, fullres_image)
Revision ID: 005
Revises: 004_widen_string_columns
Create Date: 2026-04-16
"""
from alembic import op
revision = "005_unique_images_constraint"
down_revision = "004_widen_string_columns"
branch_labels = None
depends_on = None
def upgrade() -> None:
# Удаляем дубли через CTE + ROW_NUMBER (быстрее чем NOT IN subquery)
op.execute("""
WITH dupes AS (
SELECT id,
ROW_NUMBER() OVER (PARTITION BY car_id, fullres_image ORDER BY id) AS rn
FROM images
)
DELETE FROM images WHERE id IN (SELECT id FROM dupes WHERE rn > 1)
""")
op.create_unique_constraint(
"uq_images_car_id_fullres",
"images",
["car_id", "fullres_image"],
)
def downgrade() -> None:
op.drop_constraint("uq_images_car_id_fullres", "images", type_="unique")

31
alembic/versions/006_image_car_id_bigint.py Normal file
View File

@@ -0,0 +1,31 @@
"""widen images.car_id from integer to bigint
Revision ID: 006
Revises: 005_unique_images_constraint
Create Date: 2026-04-16
"""
import sqlalchemy as sa
from alembic import op
revision = "006_image_car_id_bigint"
down_revision = "005_unique_images_constraint"
def upgrade() -> None:
op.alter_column(
"images",
"car_id",
existing_type=sa.Integer(),
type_=sa.BigInteger(),
existing_nullable=False,
)
def downgrade() -> None:
op.alter_column(
"images",
"car_id",
existing_type=sa.BigInteger(),
type_=sa.Integer(),
existing_nullable=False,
)

98
docker-compose.yml
View File

@@ -1,15 +1,18 @@
x-app-env: &app-env
IAAI_DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper}
IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-20}
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
x-app-env: &app-env
# Database (отдельная БД для Encar, порт 5433)
ENCAR_DATABASE_URL: postgresql+psycopg2://encar:encar@postgres:5432/encar_db
ENCAR_DATABASE_POOL_RECYCLE_SECONDS: ${ENCAR_DATABASE_POOL_RECYCLE_SECONDS:-1800}
# Redis / Celery (порт 6380, db=1)
ENCAR_REDIS_URL: ${ENCAR_REDIS_URL:-redis://redis:6379/1}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/1}
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/1}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-14400}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-14520}
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-15000}
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-50}
# Encar settings
ENCAR_BEAT_INTERVAL_MINUTES: ${ENCAR_BEAT_INTERVAL_MINUTES:-60}
ENCAR_BEAT_LIMIT: ${ENCAR_BEAT_LIMIT:-0}
TZ: ${TZ:-UTC}
x-env-file: &env-file
@@ -20,31 +23,23 @@ x-app-service: &app-service
build: .
env_file: *env-file
environment: *app-env
volumes:
- ./runtime_config.json:/app/runtime_config.json:ro
x-worker-service: &worker-service
<<: *app-service
volumes:
- ./runtime_config.json:/app/runtime_config.json:ro
- tokens_data:/data
services:
# PostgreSQL
# PostgreSQL (порт 5434 наружу)
postgres:
image: postgres:16-alpine
container_name: iaai-postgres
container_name: encar-postgres
restart: unless-stopped
environment:
POSTGRES_USER: iaai
POSTGRES_PASSWORD: iaai
POSTGRES_DB: iaai_scraper
POSTGRES_USER: encar
POSTGRES_PASSWORD: encar
POSTGRES_DB: encar_db
ports:
- "127.0.0.1:5432:5432"
- "127.0.0.1:5434:5432"
volumes:
- pgdata:/var/lib/postgresql/data
- encar_pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
test: ["CMD-SHELL", "pg_isready -U encar -d encar_db"]
interval: 5s
timeout: 3s
retries: 5
@@ -54,13 +49,13 @@ services:
max-size: "10m"
max-file: "5"
# Redis (Celery broker)
# Redis (порт 6380 наружу)
redis:
image: redis:7-alpine
container_name: iaai-redis
container_name: encar-redis
restart: unless-stopped
ports:
- "127.0.0.1:6379:6379"
- "127.0.0.1:6380:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
@@ -71,7 +66,7 @@ services:
--appendonly yes
--save 60 1000
volumes:
- redisdata:/data
- encar_redisdata:/data
logging:
driver: json-file
options:
@@ -81,7 +76,7 @@ services:
# DB migrations
migrate:
<<: *app-service
container_name: iaai-migrate
container_name: encar-migrate
restart: "no"
depends_on:
postgres:
@@ -91,24 +86,24 @@ services:
# FastAPI
api:
<<: *app-service
container_name: iaai-api
container_name: encar-api
restart: unless-stopped
ports:
- "127.0.0.1:8000:8000"
- "127.0.0.1:8001:8000"
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
command: >
uvicorn iaai_scraper.api.app:app
uvicorn encar_scraper.api.app:app
--host 0.0.0.0 --port 8000 --workers 1
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
start_period: 20s
stop_grace_period: 30s
logging:
driver: json-file
@@ -116,10 +111,10 @@ services:
max-size: "10m"
max-file: "5"
# Celery Worker
# Celery Worker (Encar — только HTTP API, без браузера)
worker:
<<: *worker-service
container_name: iaai-worker
<<: *app-service
container_name: encar-worker
restart: unless-stopped
init: true
depends_on:
@@ -127,18 +122,18 @@ services:
condition: service_completed_successfully
redis:
condition: service_healthy
stop_grace_period: 60s
stop_grace_period: 30s
command: >
celery -A iaai_scraper.worker.celery_app worker
--loglevel=info --concurrency=1 --pool=prefork
celery -A encar_scraper.worker.celery_app worker
--loglevel=info --concurrency=4 --pool=prefork
--pidfile=/tmp/celery-worker.pid
-Q scraping --max-tasks-per-child=20
-Q encar --max-tasks-per-child=50
healthcheck:
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
interval: 60s
timeout: 10s
retries: 3
start_period: 90s
start_period: 30s
logging:
driver: json-file
options:
@@ -147,8 +142,8 @@ services:
# Celery Beat (периодический планировщик)
beat:
<<: *worker-service
container_name: iaai-beat
<<: *app-service
container_name: encar-beat
restart: unless-stopped
init: true
depends_on:
@@ -157,7 +152,7 @@ services:
redis:
condition: service_healthy
command: >
celery -A iaai_scraper.worker.celery_app beat
celery -A encar_scraper.worker.celery_app beat
--loglevel=info
--pidfile=/tmp/celery-beat.pid
--schedule=/tmp/celerybeat-schedule
@@ -166,7 +161,7 @@ services:
interval: 60s
timeout: 10s
retries: 3
start_period: 90s
start_period: 30s
logging:
driver: json-file
options:
@@ -174,6 +169,5 @@ services:
max-file: "5"
volumes:
pgdata:
redisdata:
tokens_data:
encar_pgdata:
encar_redisdata:

0
iaai_scraper/__init__.py → encar_scraper/__init__.py
View File

0
iaai_scraper/api/__init__.py → encar_scraper/api/__init__.py
View File

4
iaai_scraper/api/app.py → encar_scraper/api/app.py
View File

@@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
_settings = settings or Settings()
app = FastAPI(
title="IAAI Scraper API",
description="REST API для управления задачами скрапинга IAAI и просмотра данных",
title="Encar Scraper API",
description="REST API для управления задачами скрапинга Encar и просмотра данных",
version="1.0.0",
lifespan=lifespan,
)

2
iaai_scraper/api/deps.py → encar_scraper/api/deps.py
View File

@@ -1,4 +1,4 @@
# Dependency helpers для FastAPI-роутов.
# Вспомогательные зависимости для FastAPI-роутов.
from fastapi import Request

0
iaai_scraper/api/routes/__init__.py → encar_scraper/api/routes/__init__.py
View File

3
iaai_scraper/api/routes/cars.py → encar_scraper/api/routes/cars.py
View File

@@ -2,6 +2,7 @@
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select
from sqlalchemy.orm import selectinload
from ..deps import get_persistence
from ...storage.db import PersistenceService
@@ -42,7 +43,7 @@ def list_cars(
offset = (page - 1) * per_page
cars = session.execute(
query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
query.options(selectinload(Car.images)).order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
).scalars().all()
return {

7
iaai_scraper/api/routes/health.py → encar_scraper/api/routes/health.py
View File

@@ -1,5 +1,7 @@
# Роут проверки доступности сервиса и соединения с БД.
import logging
from fastapi import APIRouter, Depends
from sqlalchemy import text
@@ -7,6 +9,7 @@ from ..deps import get_persistence
from ...storage.db import PersistenceService
router = APIRouter()
logger = logging.getLogger("encar_scraper.api.health")
@router.get("/health")
@@ -18,10 +21,10 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
session.execute(text("SELECT 1"))
db_ok = True
except Exception:
pass
logger.warning("Health DB check failed", exc_info=True)
return {
"status": "ok" if db_ok else "degraded",
"service": "iaai-scraper-api",
"service": "encar-scraper-api",
"database": "connected" if db_ok else "unavailable",
}

73
iaai_scraper/api/routes/tasks.py → encar_scraper/api/routes/tasks.py
View File

@@ -1,4 +1,4 @@
# Роуты запуска задач синхронизации и просмотра истории sync-runs.
# Роуты запуска задач синхронизации Encar и просмотра истории sync-runs.
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel
@@ -8,34 +8,33 @@ from ..deps import get_persistence
from ...storage.db import PersistenceService
from ...storage.models import SyncRun
from ...worker.celery_app import celery_app
from ...worker.tasks import sync_vehicle_task, sync_listing_task
from ...worker.tasks import encar_sync_listing_task, encar_sync_vehicle_task
router = APIRouter()
class SyncVehicleRequest(BaseModel):
vehicle_url: str
lane: str = "iaai"
lane: str = "encar"
class SyncListingRequest(BaseModel):
make: str | None = None
model: str | None = None
lane: str = "iaai_cars"
car_type: str = "all" # all, domestic, import
manufacturer: str | None = None
year_from: int | None = None
year_to: int | None = None
price_max: int | None = None
lane: str = "encar"
limit: int | None = None
only_new: bool | None = None
@router.post("/tasks/sync-vehicle")
def start_sync_vehicle(
body: SyncVehicleRequest,
):
# Запустить задачу скрапинга одного автомобиля через Celery
result = sync_vehicle_task.apply_async(
def start_sync_vehicle(body: SyncVehicleRequest):
"""Синхронизация одного авто Encar."""
result = encar_sync_vehicle_task.apply_async(
kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
queue="scraping",
queue="encar",
)
return {
"task_id": result.id,
"status": "queued",
@@ -44,29 +43,31 @@ def start_sync_vehicle(
@router.post("/tasks/sync-listing")
def start_sync_listing(
body: SyncListingRequest,
):
# Запустить задачу полного цикла листинга через Celery
result = sync_listing_task.apply_async(
def start_sync_listing(body: SyncListingRequest):
"""Синхронизация листинга Encar через публичный API."""
result = encar_sync_listing_task.apply_async(
kwargs={
"make": body.make,
"model": body.model,
"car_type": body.car_type,
"manufacturer": body.manufacturer,
"year_from": body.year_from,
"year_to": body.year_to,
"price_max": body.price_max,
"lane": body.lane,
"limit": body.limit,
"only_new": body.only_new,
},
queue="scraping",
queue="encar",
)
return {
"task_id": result.id,
"status": "queued",
"car_type": body.car_type,
"manufacturer": body.manufacturer,
}
@router.get("/tasks/{task_id}")
def get_task_status(task_id: str):
"""Статус задачи Celery."""
result = celery_app.AsyncResult(task_id)
payload: dict = {
@@ -90,7 +91,7 @@ def list_sync_runs(
per_page: int = Query(20, ge=1, le=100),
persistence: PersistenceService = Depends(get_persistence),
):
# История запусков синхронизации
"""История запусков синхронизации."""
with persistence.session_scope() as session:
total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
offset = (page - 1) * per_page
@@ -104,17 +105,17 @@ def list_sync_runs(
"per_page": per_page,
"items": [
{
"id": r.id,
"started_at": r.started_at.isoformat() if r.started_at else None,
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
"status": r.status,
"lane": r.lane,
"ids_fetched": r.ids_fetched,
"cars_upserted": r.cars_upserted,
"cars_failed": r.cars_failed,
"images_upserted": r.images_upserted,
"error_summary": r.error_summary,
"id": run.id,
"started_at": run.started_at.isoformat() if run.started_at else None,
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
"status": run.status,
"lane": run.lane,
"ids_fetched": run.ids_fetched,
"cars_upserted": run.cars_upserted,
"cars_failed": run.cars_failed,
"images_upserted": run.images_upserted,
"error_summary": run.error_summary,
}
for r in runs
for run in runs
],
}

103
encar_scraper/cli.py Normal file
View File

@@ -0,0 +1,103 @@
import argparse
from pathlib import Path
from .core.config import Settings
from .core.utils import save_to_json
from .encar import EncarScraper, EncarFilters, CAR_TYPE_ALL, CAR_TYPE_DOMESTIC, CAR_TYPE_IMPORT
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Encar scraper CLI")
subparsers = parser.add_subparsers(dest="command", required=True)
default_output_dir = Path("artifacts/json")
subparsers.add_parser("init-db", help="Create DB tables")
# collect-listing
collect_parser = subparsers.add_parser("collect-listing", help="Collect Encar listing")
collect_parser.add_argument("--limit", type=int, default=None, help="Max items to collect")
collect_parser.add_argument("--car-type", choices=["all", "domestic", "import"], default="all",
help="all=все, domestic=корейские, import=импорт")
collect_parser.add_argument("--manufacturer", default=None, help="Filter by manufacturer (e.g. BMW, 현대)")
collect_parser.add_argument("--year-from", type=int, default=None, help="Min year")
collect_parser.add_argument("--year-to", type=int, default=None, help="Max year")
collect_parser.add_argument("--price-max", type=int, default=None, help="Max price in 만원 (10k KRW)")
collect_parser.add_argument("--output", default=str(default_output_dir / "encar_listing.json"))
# scrape-vehicle
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Fetch single vehicle data")
scrape_parser.add_argument("vehicle_url")
scrape_parser.add_argument("--output", default=str(default_output_dir / "encar_vehicle_detail.json"))
# sync-vehicle
sync_parser = subparsers.add_parser("sync-vehicle", help="Sync single vehicle into DB")
sync_parser.add_argument("vehicle_url")
sync_parser.add_argument("--lane", default="encar")
sync_parser.add_argument("--output", default=str(default_output_dir / "encar_sync_vehicle.json"))
# sync-listing
sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing and sync all to DB")
sync_listing_parser.add_argument("--limit", type=int, default=None, help="Max items to sync")
sync_listing_parser.add_argument("--car-type", choices=["all", "domestic", "import"], default="all")
sync_listing_parser.add_argument("--manufacturer", default=None, help="Filter by manufacturer")
sync_listing_parser.add_argument("--year-from", type=int, default=None)
sync_listing_parser.add_argument("--year-to", type=int, default=None)
sync_listing_parser.add_argument("--price-max", type=int, default=None)
sync_listing_parser.add_argument("--lane", default="encar")
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default="true")
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "encar_sync_listing.json"))
return parser
def _build_filters_from_args(args) -> EncarFilters:
"""Строит фильтры из аргументов CLI."""
car_type_map = {"all": CAR_TYPE_ALL, "domestic": CAR_TYPE_DOMESTIC, "import": CAR_TYPE_IMPORT}
return EncarFilters(
car_type=car_type_map.get(getattr(args, "car_type", "all"), CAR_TYPE_ALL),
manufacturer=getattr(args, "manufacturer", None),
year_from=getattr(args, "year_from", None),
year_to=getattr(args, "year_to", None),
price_to=getattr(args, "price_max", None),
)
def main() -> None:
import sys
import logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
stream=sys.stderr,
)
parser = build_parser()
args = parser.parse_args()
scraper = EncarScraper()
if args.command == "init-db":
result = scraper.init_db()
print(f"DB initialized: {result}")
return
elif args.command == "collect-listing":
filters = _build_filters_from_args(args)
result = scraper.collect_listing(limit=args.limit, filters=filters)
elif args.command == "scrape-vehicle":
result = scraper.scrape_vehicle_detail(args.vehicle_url)
elif args.command == "sync-vehicle":
result = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
elif args.command == "sync-listing":
filters = _build_filters_from_args(args)
only_new = args.only_new == "true"
result = scraper.sync_listing(limit=args.limit, filters=filters, lane=args.lane, only_new=only_new)
else:
raise SystemExit(f"Unknown command: {args.command}")
save_to_json(result, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
if __name__ == "__main__":
main()

0
iaai_scraper/core/__init__.py → encar_scraper/core/__init__.py
View File

106
encar_scraper/core/config.py Normal file
View File

@@ -0,0 +1,106 @@
import os
from dataclasses import dataclass, field
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
TRUE_VALUES = {"1", "true", "yes", "on"}
# Хелперы для чтения env-переменных с приведением типов
def _env_str(name: str, default: str) -> str:
value = os.getenv(name)
return value if value is not None else default
def _env_optional_str(name: str) -> str | None:
value = os.getenv(name)
if value is None:
return None
value = value.strip()
return value or None
def _env_path_str(name: str) -> str | None:
value = _env_optional_str(name)
if value is None:
return None
return str(Path(value).expanduser())
def _env_bool(name: str, default: bool) -> bool:
fallback = "true" if default else "false"
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
def _env_int(name: str, default: int) -> int:
return int(_env_str(name, str(default)).strip())
def _env_float(name: str, default: float) -> float:
return float(_env_str(name, str(default)).strip())
# Конфиг браузерного отпечатка (User-Agent для HTTP запросов)
@dataclass(slots=True)
class FingerprintConfig:
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/135.0.0.0 Safari/537.36"
)
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
@dataclass(slots=True)
class DatabaseConfig:
url: str = _env_str("ENCAR_DATABASE_URL", "postgresql+psycopg2://encar:encar@localhost:5434/encar_db")
echo: bool = _env_bool("ENCAR_DATABASE_ECHO", False)
pool_size: int = _env_int("ENCAR_DATABASE_POOL_SIZE", 5)
max_overflow: int = _env_int("ENCAR_DATABASE_MAX_OVERFLOW", 10)
pool_recycle_seconds: int = _env_int("ENCAR_DATABASE_POOL_RECYCLE_SECONDS", 1800)
auto_create_tables: bool = _env_bool("ENCAR_DATABASE_AUTO_CREATE_TABLES", False)
# --- Конфиг Redis (URL для Celery broker) ---
@dataclass(slots=True)
class RedisConfig:
url: str = _env_str("ENCAR_REDIS_URL", "redis://localhost:6380/1")
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@dataclass(slots=True)
class CeleryConfig:
broker_url: str = _env_str("CELERY_BROKER_URL", "")
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 14400)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 14520)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 50)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
# Encar settings
encar_beat_interval_minutes: int = _env_int("ENCAR_BEAT_INTERVAL_MINUTES", 60)
encar_beat_limit: int | None = _env_int("ENCAR_BEAT_LIMIT", 0) or None
# --- Главный объект настроек: собирает все блоки конфигурации ---
@dataclass(slots=True)
class Settings:
log_level: str = _env_str("ENCAR_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("ENCAR_LOG_FILE")
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
# Глобальный синглтон — используется по умолчанию во всех модулях.
settings = Settings()

2
encar_scraper/core/exceptions.py Normal file
View File

@@ -0,0 +1,2 @@
class ScraperError(Exception):
"""Базовое исключение скрапера."""

2
iaai_scraper/core/logs.py → encar_scraper/core/logs.py
View File

@@ -2,7 +2,7 @@ import logging
import sys
from contextvars import ContextVar
# ContextVar хранит trace_id текущего потока/корутины.
# Храним trace_id текущего потока/корутины.
TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")

97
encar_scraper/core/runtime_config.py Normal file
View File

@@ -0,0 +1,97 @@
"""Загрузчик runtime_config.json для управления фильтрами синхронизации."""
from __future__ import annotations
import json
import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
logger = logging.getLogger("encar_scraper.runtime_config")
DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent.parent / "runtime_config.json"
@dataclass
class SyncConfig:
name: str | None = None
lane: str = "encar_cars"
only_new: bool = False
limit: int | None = None
probe_all_photos: bool = False
@dataclass
class FiltersConfig:
price_min: int | None = None
price_max: int | None = None
mileage_min: int | None = None
mileage_max: int | None = None
brands: list[str] = field(default_factory=list)
models: list[str] = field(default_factory=list)
years: list[int] = field(default_factory=list)
body_types: list[str] = field(default_factory=list)
colors: list[str] = field(default_factory=list)
drives: list[str] = field(default_factory=list)
gearboxes: list[str] = field(default_factory=list)
exclude_brands: list[str] = field(default_factory=list)
exclude_models: list[str] = field(default_factory=list)
exclude_years: list[int] = field(default_factory=list)
exclude_body_types: list[str] = field(default_factory=list)
@dataclass
class RuntimeConfig:
sync: SyncConfig = field(default_factory=SyncConfig)
filters: FiltersConfig = field(default_factory=FiltersConfig)
def load_runtime_config(path: str | Path | None = None) -> RuntimeConfig:
"""Загружает runtime_config.json. Если файл не найден — возвращает дефолт."""
config_path = Path(path) if path else DEFAULT_CONFIG_PATH
if not config_path.exists():
logger.info("runtime_config.json not found at %s, using defaults", config_path)
return RuntimeConfig()
try:
raw: dict[str, Any] = json.loads(config_path.read_text(encoding="utf-8"))
except Exception as exc:
logger.warning("Failed to parse runtime_config.json: %s, using defaults", exc)
return RuntimeConfig()
sync_raw = raw.get("sync") or {}
filters_raw = raw.get("filters") or {}
price_raw = filters_raw.get("price") or {}
mileage_raw = filters_raw.get("mileage") or {}
sync = SyncConfig(
name=sync_raw.get("name"),
lane=sync_raw.get("lane", "encar_cars"),
only_new=bool(sync_raw.get("only_new", False)),
limit=sync_raw.get("limit"),
probe_all_photos=bool(sync_raw.get("probe_all_photos", False)),
)
filters = FiltersConfig(
price_min=price_raw.get("min"),
price_max=price_raw.get("max"),
mileage_min=mileage_raw.get("min"),
mileage_max=mileage_raw.get("max"),
brands=[b.lower() for b in (filters_raw.get("brands") or [])],
models=[m.lower() for m in (filters_raw.get("models") or [])],
years=filters_raw.get("years") or [],
body_types=[b.lower() for b in (filters_raw.get("body_types") or [])],
colors=[c.lower() for c in (filters_raw.get("colors") or [])],
drives=[d.lower() for d in (filters_raw.get("drives") or [])],
gearboxes=[g.lower() for g in (filters_raw.get("gearboxes") or [])],
exclude_brands=[b.lower() for b in (filters_raw.get("exclude_brands") or [])],
exclude_models=[m.lower() for m in (filters_raw.get("exclude_models") or [])],
exclude_years=filters_raw.get("exclude_years") or [],
exclude_body_types=[b.lower() for b in (filters_raw.get("exclude_body_types") or [])],
)
logger.info(
"runtime_config loaded: lane=%s, limit=%s, brands=%d, exclude_brands=%d",
sync.lane, sync.limit, len(filters.brands), len(filters.exclude_brands),
)
return RuntimeConfig(sync=sync, filters=filters)

37
iaai_scraper/core/utils.py → encar_scraper/core/utils.py
View File

@@ -1,7 +1,7 @@
import json
import re
from pathlib import Path
from typing import Any, Iterable
from typing import Any, Callable, Iterable
def save_to_json(data: Any, filename: str | Path) -> None:
@@ -17,7 +17,7 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
return None
# Regex для VIN, lot, price.
# Регулярные выражения для VIN, lot и price.
VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
LOT_RE = re.compile(r"\b(\d{7,10})\b")
PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
@@ -37,3 +37,36 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
for item in obj:
found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
return found
def deep_find_all_keys(
payloads: list,
field_map: dict[str, set[str]],
max_depth: int = 64,
) -> dict[str, list]:
"""Извлекает все нужные поля за один проход по JSON."""
# Готовим обратную карту: нормализованный ключ -> имя поля.
reverse: dict[str, str] = {}
for field_name, keys in field_map.items():
for k in keys:
reverse[k.lower()] = field_name
result: dict[str, list] = {f: [] for f in field_map}
def _recurse(obj: Any, depth: int) -> None:
if depth >= max_depth:
return
if isinstance(obj, dict):
for k, v in obj.items():
field = reverse.get(k.lower())
if field is not None:
result[field].append(v)
_recurse(v, depth + 1)
elif isinstance(obj, list):
for item in obj:
_recurse(item, depth + 1)
for payload in payloads:
_recurse(payload, 0)
return result

1653
encar_scraper/encar.py Normal file
View File

File diff suppressed because it is too large Load Diff

0
iaai_scraper/parsing/__init__.py → encar_scraper/storage/__init__.py
View File

494
encar_scraper/storage/db.py Normal file
View File

@@ -0,0 +1,494 @@
import logging
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Any, Iterator
from sqlalchemy import create_engine, delete, or_, select, text, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session, sessionmaker
from ..core.config import Settings
from .models import Base, Car, Image, SyncRun
from .schemas import CarRecord
logger = logging.getLogger("encar_scraper.db")
CAR_DB_FIELDS = {
col.key for col in Car.__table__.columns
if col.key not in ("id",)
}
_IN_CHUNK_SIZE = 5000
class PersistenceService:
def __init__(self, settings: Settings) -> None:
self.settings = settings
engine_kwargs = {
"echo": settings.database.echo,
"future": True,
}
if "postgresql" in settings.database.url:
engine_kwargs["pool_size"] = settings.database.pool_size
engine_kwargs["max_overflow"] = settings.database.max_overflow
engine_kwargs["pool_pre_ping"] = True
engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
self.engine = create_engine(settings.database.url, **engine_kwargs)
self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
def create_tables(self) -> None:
# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
is_sqlite = self.settings.database.url.startswith("sqlite")
if not is_sqlite and not self.settings.database.auto_create_tables:
return
try:
Base.metadata.create_all(self.engine)
except Exception:
logger.debug("create_tables skipped (schema already exists)")
@contextmanager
def session_scope(self) -> Iterator[Session]:
session = self.session_factory()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
def start_sync_run(self, lane: str) -> int:
with self.session_scope() as session:
now = datetime.now(timezone.utc)
stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
for stale in stale_runs:
stale.status = "failed"
stale.finished_at = now
if not stale.error_summary:
stale.error_summary = "Recovered stale running sync run before starting a new run"
run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
session.add(run)
session.flush()
return int(run.id)
def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
with self.session_scope() as session:
run = session.get(SyncRun, run_id)
if run is None:
return
run.finished_at = datetime.now(timezone.utc)
run.status = status
run.ids_fetched = ids_fetched
run.cars_upserted = cars_upserted
run.cars_failed = cars_failed
run.images_upserted = images_upserted
run.error_summary = error_summary
@staticmethod
def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
if not images:
return
session.add_all([
Image(
fullres_image=str(img["fullres_image"]),
preview_image=str(img["preview_image"]),
order_index=int(img.get("order_index", 0)),
car_id=car_id,
)
for img in images
])
@staticmethod
def _car_payload(record: CarRecord) -> dict[str, object]:
payload = record.model_dump(mode="python")
return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
def _is_postgres(self) -> bool:
return self.engine.dialect.name == "postgresql"
def _load_existing_cars(
self,
session: Session,
origin_ids: list[str],
origin_urls: list[str],
) -> tuple[dict[str, Car], dict[str, Car]]:
existing_by_id: dict[str, Car] = {}
existing_by_url: dict[str, Car] = {}
if not origin_ids and not origin_urls:
return existing_by_id, existing_by_url
existing_cars: list[Car] = []
max_len = max(len(origin_ids), len(origin_urls), 1)
for i in range(0, max_len, _IN_CHUNK_SIZE):
id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
conditions = []
if id_chunk:
conditions.append(Car.origin_id.in_(id_chunk))
if url_chunk:
conditions.append(Car.origin_url.in_(url_chunk))
if not conditions:
continue
rows = session.execute(
select(Car).where(or_(*conditions))
).scalars().all()
existing_cars.extend(rows)
for car in existing_cars:
if car.origin_id:
existing_by_id[car.origin_id] = car
if car.origin_url:
existing_by_url[car.origin_url] = car
return existing_by_id, existing_by_url
def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
existing_images_map: dict[int, set[str]] = {}
if not car_ids:
return existing_images_map
car_id_list = list(car_ids)
for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
img_rows = session.execute(
select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
).all()
for cid, furl in img_rows:
existing_images_map.setdefault(int(cid), set()).add(str(furl))
return existing_images_map
@staticmethod
def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
return {
key: getattr(insert_stmt.excluded, key)
for key in CAR_DB_FIELDS
}
def _replace_images_for_car(
self,
session: Session,
car_id: int,
images: list[dict[str, object]],
origin_id: str,
) -> int:
nested = session.begin_nested()
try:
session.execute(delete(Image).where(Image.car_id == car_id))
self._add_images(session, car_id, images)
session.flush()
nested.commit()
return len(images)
except Exception:
nested.rollback()
logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
return 0
def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
inserted = 0
updated = 0
images_total = 0
with self.session_scope() as session:
origin_ids = [r.origin_id for r in records if r.origin_id]
origin_urls = [r.origin_url for r in records if r.origin_url]
existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
entries: list[dict[str, object]] = []
upsert_payloads: list[dict[str, object]] = []
for record in records:
payload = self._car_payload(record)
images = [image.model_dump(mode="python") for image in record.images]
car_by_id = existing_by_id.get(record.origin_id)
car_by_url = existing_by_url.get(record.origin_url)
entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
for key, value in payload.items():
setattr(car_by_url, key, value)
car_by_url.last_seen_at = record.last_seen_at
entry["car_id"] = int(car_by_url.id)
updated += 1
else:
upsert_payloads.append(payload)
if car_by_id is not None:
updated += 1
else:
inserted += 1
entries.append(entry)
session.flush()
if upsert_payloads:
insert_stmt = pg_insert(Car).values(upsert_payloads)
upsert_stmt = insert_stmt.on_conflict_do_update(
index_elements=[Car.origin_id],
set_=self._postgres_upsert_set_map(insert_stmt),
).returning(Car.id, Car.origin_id)
rows = session.execute(upsert_stmt).all()
car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
for entry in entries:
if entry["car_id"] is not None:
continue
record = entry["record"]
car_id = car_ids_by_origin_id.get(record.origin_id)
if car_id is None:
raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
entry["car_id"] = car_id
car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
existing_images_map = self._load_existing_image_urls(session, car_ids)
images_by_car_id: dict[int, list[dict[str, object]]] = {}
replace_ids: list[int] = []
for entry in entries:
car_id = int(entry["car_id"])
images = entry["images"]
new_image_urls = {
str(img.get("fullres_image", ""))
for img in images
if img.get("fullres_image")
}
old_image_urls = existing_images_map.get(car_id, set())
if new_image_urls != old_image_urls:
replace_ids.append(car_id)
images_by_car_id[car_id] = images
else:
images_total += len(old_image_urls)
if replace_ids:
for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
for car_id in replace_ids:
images = images_by_car_id[car_id]
self._add_images(session, car_id, images)
images_total += len(images)
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
def upsert_car(self, record: CarRecord):
# Вставка или обновление автомобиля по origin_id/origin_url.
payload = self._car_payload(record)
images = [image.model_dump(mode="python") for image in record.images]
with self.session_scope() as session:
if self._is_postgres():
car_by_url = session.execute(
select(Car).where(Car.origin_url == record.origin_url)
).scalar_one_or_none()
if car_by_url is not None and car_by_url.origin_id != record.origin_id:
for key, value in payload.items():
setattr(car_by_url, key, value)
car_by_url.last_seen_at = record.last_seen_at
session.flush()
car_id = int(car_by_url.id)
action = "updated"
else:
existed = session.execute(
select(Car.id).where(Car.origin_id == record.origin_id)
).scalar_one_or_none() is not None
insert_stmt = pg_insert(Car).values(**payload)
upsert_stmt = insert_stmt.on_conflict_do_update(
index_elements=[Car.origin_id],
set_=self._postgres_upsert_set_map(insert_stmt),
).returning(Car.id)
car_id = int(session.execute(upsert_stmt).scalar_one())
action = "updated" if existed or car_by_url is not None else "inserted"
images_upserted = self._replace_images_for_car(
session, car_id, images, record.origin_id,
)
return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
car = session.execute(
select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
).scalar_one_or_none()
action = "inserted"
if car is None:
car = Car(**payload)
session.add(car)
session.flush()
else:
action = "updated"
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
session.flush()
images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
self._add_images(session, int(car.id), images)
session.flush()
return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
"""Пакетный upsert нескольких автомобилей в одной транзакции.
Оптимизации:
- Дедупликация записей по origin_id перед вставкой.
- Chunked IN-queries для больших списков (обход лимита PG параметров).
- Пропуск перезаписи изображений, если набор URL не изменился.
- Один DELETE по car_id IN (...) вместо удаления по одному.
- Fallback на по-одному upsert если batch commit упал.
"""
# ── Дедупликация записей внутри батча ──
seen_ids: dict[str, int] = {}
unique_records: list[CarRecord] = []
for idx, r in enumerate(records):
key = r.origin_id or r.origin_url
if key in seen_ids:
logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
continue
seen_ids[key] = idx
unique_records.append(r)
if len(unique_records) < len(records):
logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
records = unique_records
try:
return self._upsert_cars_batch_inner(records)
except Exception as exc:
logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
return self._upsert_cars_individually(records)
def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
"""Внутренняя реализация batched upsert (одна транзакция)."""
if self._is_postgres():
return self._upsert_cars_batch_postgres(records)
inserted = 0
updated = 0
images_total = 0
with self.session_scope() as session:
# Получаем существующие записи chunked-запросами.
origin_ids = [r.origin_id for r in records if r.origin_id]
origin_urls = [r.origin_url for r in records if r.origin_url]
existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
existing_car_ids = set()
for record in records:
car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
if car is not None:
existing_car_ids.add(int(car.id))
# Строим маппинг car_id → set(image_urls) для сравнения.
existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
new_cars: list[tuple[Car, list[dict]]] = []
update_cars_needing_images: list[tuple[Car, list[dict]]] = []
for record in records:
payload = self._car_payload(record)
images = [image.model_dump(mode="python") for image in record.images]
car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
if car is None:
car = Car(**payload)
session.add(car)
inserted += 1
new_cars.append((car, images))
else:
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
updated += 1
# Проверяем, изменились ли изображения.
new_image_urls = {img.get("fullres_image", "") for img in images}
old_image_urls = existing_images_map.get(int(car.id), set())
if new_image_urls != old_image_urls:
update_cars_needing_images.append((car, images))
else:
images_total += len(old_image_urls)
# Один flush для всех вставок.
session.flush()
# Добавляем изображения для новых автомобилей.
for car, images in new_cars:
self._add_images(session, int(car.id), images)
images_total += len(images)
# Массово обновляем изображения только для машин с изменёнными картинками.
if update_cars_needing_images:
update_ids = [int(car.id) for car, _ in update_cars_needing_images]
for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
chunk = update_ids[i:i + _IN_CHUNK_SIZE]
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
for car, images in update_cars_needing_images:
self._add_images(session, int(car.id), images)
images_total += len(images)
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
# Fallback на поштучный upsert.
inserted = 0
updated = 0
images_total = 0
for record in records:
try:
result = self.upsert_car(record)
action = result.get("action", "inserted")
if action == "inserted":
inserted += 1
else:
updated += 1
images_total += int(result.get("images_upserted", 0))
except Exception as exc:
logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "encar") -> int:
"""Помечает авто как проданные, если их нет в активном листинге (по origin_id).
Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
что кардинально быстрее при больших объёмах (200K+ IDs).
"""
if not active_origin_ids:
return 0
with self.session_scope() as session:
if self._is_postgres():
session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_ids (origin_id TEXT NOT NULL) ON COMMIT DROP"))
session.execute(text("TRUNCATE _active_ids"))
id_list = list(active_origin_ids)
for i in range(0, len(id_list), _IN_CHUNK_SIZE):
chunk = id_list[i:i + _IN_CHUNK_SIZE]
values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
params = {f"u{j}": oid for j, oid in enumerate(chunk)}
session.execute(text(f"INSERT INTO _active_ids (origin_id) VALUES {values}"), params)
session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_ids ON _active_ids (origin_id)"))
result = session.execute(text("""
UPDATE cars
SET is_sold = TRUE
FROM (
SELECT c.id
FROM cars c
LEFT JOIN _active_ids a ON c.origin_id = a.origin_id
WHERE a.origin_id IS NULL
AND c.is_sold = FALSE
AND c.origin_id LIKE 'encar:%%'
) sub
WHERE cars.id = sub.id
"""))
count = result.rowcount or 0
else:
stmt = (
update(Car)
.where(Car.origin_id.notin_(active_origin_ids))
.where(Car.is_sold == False) # noqa: E712
.where(Car.origin_id.like("encar:%"))
.values(is_sold=True)
)
result = session.execute(stmt)
count = result.rowcount or 0
if count:
logger.info("Marked %d cars as sold (no longer in listing)", count)
return count

21
iaai_scraper/storage/enums.py → encar_scraper/storage/enums.py
View File

@@ -13,21 +13,24 @@ BODY_TYPE_ENUM_VALUES = (
"TRUCK",
"OPEN",
"RV",
"VAN",
"CONVERTIBLE",
"BUS",
"COMPACT",
"MID_SIZE",
"FULL_SIZE",
"SPORTS",
"CLASSIC",
"ELECTRIC",
"HYBRID",
"SPECIAL",
"WAGON",
"OTHER",
"NA",
)
COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
ORIGIN_ENUM_VALUES = (
"TAU",
"CARSENSOR",
"HANAMARU",
"ENCAR",
"KURUMA_TRADER",
"ASNET",
"KABABA",
"ACV",
"COPART",
"IAAI",
"NA",
)
SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

25
iaai_scraper/storage/models.py → encar_scraper/storage/models.py
View File

@@ -23,28 +23,29 @@ class Car(Base):
__tablename__ = "cars"
__table_args__ = (
Index("ix_cars_brand_model", "brand", "model"),
Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
model: Mapped[str] = mapped_column(String(50), nullable=False)
parser_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
brand: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
model: Mapped[str] = mapped_column(String(500), nullable=False)
year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD")
currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True)
gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True)
steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True)
body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER")
drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True)
steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True)
body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER")
engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
@@ -63,7 +64,7 @@ class Image(Base):
fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
preview_image: Mapped[str] = mapped_column(String(), nullable=False)
order_index: Mapped[int] = mapped_column(Integer, nullable=False)
car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
car_id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
car: Mapped[Car] = relationship("Car", back_populates="images")

0
iaai_scraper/storage/schemas.py → encar_scraper/storage/schemas.py
View File

0
iaai_scraper/worker/__init__.py → encar_scraper/worker/__init__.py
View File

35
iaai_scraper/worker/celery_app.py → encar_scraper/worker/celery_app.py
View File

@@ -1,4 +1,4 @@
# Инициализация Celery-приложения и периодических задач.
# Инициализация Celery-приложения и периодических задач для Encar.
from celery import Celery
@@ -14,7 +14,7 @@ def _result_backend() -> str:
celery_app = Celery(
"iaai_scraper",
"encar_scraper",
broker=_broker_url(),
backend=_result_backend(),
)
@@ -39,18 +39,33 @@ celery_app.conf.update(
"visibility_timeout": settings.celery.broker_visibility_timeout,
},
result_expires=86400,
worker_hijack_root_logger=False,
beat_schedule={
"periodic-sync-listing": {
"task": "iaai_scraper.worker.tasks.sync_listing_task",
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
"periodic-encar-sync": {
"task": "encar_scraper.worker.tasks.encar_sync_listing_task",
"schedule": settings.celery.encar_beat_interval_minutes * 60.0,
"args": (),
"kwargs": {"limit": settings.celery.beat_sync_limit},
"options": {"queue": "scraping"},
}
"kwargs": {
"car_type": "all",
},
"options": {"queue": "encar"},
},
},
task_routes={
"iaai_scraper.worker.tasks.*": {"queue": "scraping"}
"encar_scraper.worker.tasks.*": {"queue": "encar"},
},
)
celery_app.autodiscover_tasks(["iaai_scraper.worker"])
celery_app.autodiscover_tasks(["encar_scraper.worker"])
from celery.signals import setup_logging as celery_setup_logging
@celery_setup_logging.connect
def _configure_logging(loglevel, logfile, format, colorize, **kwargs):
"""Перехватываем настройку логирования Celery, чтобы использовать свой формат."""
from ..core.logs import setup_logging
import logging
level_name = logging.getLevelName(loglevel) if isinstance(loglevel, int) else str(loglevel)
setup_logging(level=level_name, log_file=logfile)

244
encar_scraper/worker/tasks.py Normal file
View File

@@ -0,0 +1,244 @@
# Celery tasks для синхронизации автомобилей Encar.
import logging
from threading import Event, Thread
import uuid
from celery import shared_task
from redis import Redis
from ..core.config import Settings
from ..core.runtime_config import load_runtime_config
from ..encar import EncarScraper, EncarFilters, expand_allowed_brands
from ..storage.db import PersistenceService
logger = logging.getLogger("encar_scraper.worker.tasks")
ENCAR_SYNC_LOCK_KEY = "encar:locks:sync_listing"
_persistence: PersistenceService | None = None
_redis: Redis | None = None
def _sync_listing_lock_ttl_seconds() -> int:
settings = Settings()
return max(settings.celery.task_time_limit + 120, 300)
def _get_persistence() -> PersistenceService:
global _persistence
if _persistence is None:
_persistence = PersistenceService(Settings())
return _persistence
def _get_redis() -> Redis:
global _redis
if _redis is None:
settings = Settings()
_redis = Redis.from_url(
settings.redis.url,
decode_responses=True,
socket_timeout=10,
socket_connect_timeout=5,
)
return _redis
def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
try:
return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
except Exception as exc:
logger.warning("Failed to acquire lock %s", key, exc_info=True)
return False
_REFRESH_LOCK_SCRIPT = """
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
end
return 0
"""
_RELEASE_LOCK_SCRIPT = """
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('DEL', KEYS[1])
end
return 0
"""
def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None:
try:
script = redis_client.register_script(_REFRESH_LOCK_SCRIPT)
refreshed = script(keys=[key], args=[owner_token, int(ttl_seconds)])
return bool(refreshed)
except Exception as exc:
logger.warning("Failed to refresh lock %s", key, exc_info=True)
return None
def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None:
try:
script = redis_client.register_script(_RELEASE_LOCK_SCRIPT)
script(keys=[key], args=[owner_token])
except Exception as exc:
logger.warning("Failed to release lock %s", key, exc_info=True)
def _start_lock_heartbeat(
redis_client: Redis,
key: str,
owner_token: str,
ttl_seconds: int,
) -> tuple[Event, Thread]:
stop_event = Event()
interval_seconds = max(5.0, min(30.0, ttl_seconds / 3))
def _heartbeat() -> None:
while not stop_event.wait(interval_seconds):
refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds)
if refreshed is False:
logger.warning("Lost sync_listing lock ownership for %s", owner_token)
return
thread = Thread(target=_heartbeat, name="encar-lock-heartbeat", daemon=True)
thread.start()
return stop_event, thread
@shared_task(
name="encar_scraper.worker.tasks.encar_sync_listing_task",
bind=True,
max_retries=2,
default_retry_delay=60,
acks_late=True,
)
def encar_sync_listing_task(
self,
car_type: str = "all",
manufacturer: str | None = None,
year_from: int | None = None,
year_to: int | None = None,
price_max: int | None = None,
lane: str = "encar",
limit: int | None = None,
):
"""
Полная синхронизация листинга Encar.
Собирает ВСЕ авто, обновляет цены/данные, помечает проданные.
"""
persistence = _get_persistence()
persistence.create_tables()
task_id = self.request.id or "unknown"
owner_token = f"{task_id}:{uuid.uuid4().hex}"
redis_client = _get_redis()
lock_ttl = _sync_listing_lock_ttl_seconds()
lock_acquired = _acquire_lock(redis_client, ENCAR_SYNC_LOCK_KEY, owner_token, lock_ttl)
if not lock_acquired:
logger.info("encar_sync_listing_task skipped: another sync is already running")
return {
"status": "skipped",
"reason": "sync_already_running",
"task_id": task_id,
}
heartbeat_stop: Event | None = None
heartbeat_thread: Thread | None = None
try:
heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
redis_client,
ENCAR_SYNC_LOCK_KEY,
owner_token,
lock_ttl,
)
self.update_state(state="STARTED", meta={"stage": "encar_sync_started", "task_id": task_id})
# --- Загрузка runtime_config ---
rc = load_runtime_config()
effective_limit = limit or rc.sync.limit
effective_lane = lane or rc.sync.lane or "encar"
car_type_map = {"all": "A", "domestic": "Y", "import": "N"}
filters = EncarFilters(
car_type=car_type_map.get(car_type, "A"),
manufacturer=manufacturer,
year_from=year_from,
year_to=year_to,
price_to=price_max,
)
# Набор разрешённых брендов (lowercased) из runtime_config
# Расширяем английские имена alias'ами (корейский + русский перевод)
allowed_brands = expand_allowed_brands(
set(rc.filters.brands) if rc.filters.brands else None
)
excluded_brands = expand_allowed_brands(
set(rc.filters.exclude_brands) if rc.filters.exclude_brands else None
)
scraper = EncarScraper()
result = scraper.sync_listing(
limit=effective_limit,
filters=filters,
lane=effective_lane,
redis_client=redis_client,
allowed_brands=allowed_brands,
excluded_brands=excluded_brands,
probe_all_photos=rc.sync.probe_all_photos,
)
summary = {
"task_id": task_id,
"source": "encar",
"total_available": result.get("total_available", 0),
"items_collected": result.get("items_collected", 0),
"cars_synced": result.get("cars_synced", 0),
"cars_failed": result.get("cars_failed", 0),
"cars_marked_sold": result.get("cars_marked_sold", 0),
}
logger.info(
"encar_sync_listing_task completed: %d synced, %d failed, %d marked sold",
summary["cars_synced"], summary["cars_failed"], summary["cars_marked_sold"],
)
return {"status": "success", **summary}
except Exception as exc:
logger.error("encar_sync_listing_task failed: %s", exc, exc_info=True)
raise self.retry(exc=exc)
finally:
if heartbeat_stop is not None:
heartbeat_stop.set()
if heartbeat_thread is not None:
heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
if lock_acquired:
_release_lock_if_owner(redis_client, ENCAR_SYNC_LOCK_KEY, owner_token)
@shared_task(
name="encar_scraper.worker.tasks.encar_sync_vehicle_task",
bind=True,
max_retries=2,
default_retry_delay=30,
acks_late=True,
)
def encar_sync_vehicle_task(self, vehicle_url: str, lane: str = "encar"):
"""Синхронизация одного авто Encar."""
persistence = _get_persistence()
persistence.create_tables()
try:
scraper = EncarScraper()
result = scraper.sync_vehicle(vehicle_url, lane=lane)
logger.info("encar_sync_vehicle_task completed: %s", vehicle_url)
return {
"status": "success",
"vehicle_url": vehicle_url,
"origin_id": result.get("origin_id"),
}
except Exception as exc:
logger.error("encar_sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True)
raise self.retry(exc=exc)

98
entrypoint.sh
View File

@@ -1,101 +1,7 @@
#!/bin/bash
set -euo pipefail
is_worker_command() {
local joined="$*"
case "$joined" in
*"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
return 0
;;
esac
return 1
}
is_scrape_cli_command() {
local joined="$*"
case "$joined" in
*"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
return 0
;;
esac
return 1
}
needs_browser_runtime() {
is_worker_command "$@" || is_scrape_cli_command "$@"
}
start_proxy_bridge_if_needed() {
if ! needs_browser_runtime "$@"; then
return 0
fi
if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
return 0
fi
echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
fi
echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
python -m iaai_scraper.proxy_bridge &
BRIDGE_PID=$!
sleep 1
if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
exit 1
fi
echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
}
start_xvfb_if_needed() {
if ! needs_browser_runtime "$@"; then
return 0
fi
# IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display
export DISPLAY=:99
export IAAI_HEADLESS=false
local xvfb_pid_file="/tmp/xvfb-99.pid"
if [ -f "${xvfb_pid_file}" ]; then
local existing_pid
existing_pid="$(cat "${xvfb_pid_file}")"
if [ -n "${existing_pid}" ] && kill -0 "${existing_pid}" 2>/dev/null; then
echo "[entrypoint] Reusing existing Xvfb on DISPLAY=${DISPLAY} (PID ${existing_pid})"
return 0
fi
echo "[entrypoint] Removing stale Xvfb pid file"
rm -f "${xvfb_pid_file}"
fi
if [ -S /tmp/.X11-unix/X99 ] || [ -f /tmp/.X99-lock ]; then
echo "[entrypoint] Removing stale Xvfb socket/lock files for DISPLAY=${DISPLAY}"
rm -f /tmp/.X11-unix/X99 /tmp/.X99-lock
fi
Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
XVFB_PID=$!
echo "${XVFB_PID}" > "${xvfb_pid_file}"
sleep 0.5
if ! kill -0 "${XVFB_PID}" 2>/dev/null; then
echo "[entrypoint] ERROR: Xvfb failed to start on DISPLAY=${DISPLAY}"
rm -f "${xvfb_pid_file}"
exit 1
fi
echo "[entrypoint] Xvfb started (PID ${XVFB_PID}, DISPLAY=${DISPLAY})"
}
start_proxy_bridge_if_needed "$@"
start_xvfb_if_needed "$@"
# Encar scraper — простой entrypoint без браузера.
# Просто запускает переданную команду.
exec "$@"

6
iaai_scraper/browser/__init__.py
View File

@@ -1,6 +0,0 @@
from .factory import BrowserFactory
from .listing import ListingCollector
from .network import NetworkCapture
from .pace import HumanPacer
__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]

117
iaai_scraper/browser/factory.py
View File

@@ -1,117 +0,0 @@
import json
import logging
import random
from playwright.sync_api import Browser, BrowserContext, Playwright
from ..core.config import Settings
logger = logging.getLogger("iaai_scraper.browser")
def _build_init_script() -> str:
# JS-патч признаков автоматизации.
hardware_concurrency = random.choice([4, 8, 12, 16])
device_memory = random.choice([4, 8, 16])
languages = ["en-US", "en"]
return f"""
(() => {{
const define = (obj, prop, value) => {{
try {{
Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
}} catch (e) {{}}
}};
define(navigator, 'webdriver', undefined);
define(navigator, 'platform', 'Win32');
define(navigator, 'vendor', 'Google Inc.');
define(navigator, 'language', '{languages[0]}');
define(navigator, 'languages', {json.dumps(languages)});
define(navigator, 'hardwareConcurrency', {hardware_concurrency});
define(navigator, 'deviceMemory', {device_memory});
define(navigator, 'maxTouchPoints', 0);
if (!window.chrome) {{
Object.defineProperty(window, 'chrome', {{
value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
configurable: true
}});
}}
const originalQuery = navigator.permissions && navigator.permissions.query;
if (originalQuery) {{
navigator.permissions.query = (params) => (
params && params.name === 'notifications'
? Promise.resolve({{ state: Notification.permission }})
: originalQuery(params)
);
}}
const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
WebGLRenderingContext.prototype.getParameter = function(parameter) {{
if (parameter === 37445) return 'Intel Inc.';
if (parameter === 37446) return 'Intel Iris OpenGL Engine';
return originalGetParameter.call(this, parameter);
}};
}})();
"""
class BrowserFactory:
def __init__(self, settings: Settings) -> None:
self.settings = settings
def create_browser(self, playwright: Playwright) -> Browser:
launch_kwargs: dict = {
"headless": self.settings.headless,
"args": [
"--disable-blink-features=AutomationControlled",
"--no-default-browser-check",
"--disable-dev-shm-usage",
"--disable-features=IsolateOrigins,site-per-process",
],
}
proxy_dict = self.settings.proxy.to_playwright_dict()
if proxy_dict:
launch_kwargs["proxy"] = proxy_dict
logger.info("Using proxy: %s", self.settings.proxy.server)
try:
logger.info("Trying to launch real Chrome channel")
return playwright.chromium.launch(channel="chrome", **launch_kwargs)
except Exception:
logger.warning("Chrome channel launch failed, falling back to Chromium")
return playwright.chromium.launch(**launch_kwargs)
def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
viewport = random.choice(self.settings.fingerprint.viewport_presets)
timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
color_scheme = random.choice(["light", "dark"])
context = browser.new_context(
storage_state=storage_state or None,
user_agent=self.settings.fingerprint.user_agent,
viewport=viewport,
screen=viewport,
device_scale_factor=random.choice([1, 1.25]),
is_mobile=False,
has_touch=False,
locale=self.settings.fingerprint.locale,
timezone_id=timezone_id,
color_scheme=color_scheme,
java_script_enabled=True,
ignore_https_errors=False,
extra_http_headers={
"Accept-Language": "en-US,en;q=0.9",
"DNT": "1",
"Upgrade-Insecure-Requests": "1",
"Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
"Sec-CH-UA-Mobile": "?0",
"Sec-CH-UA-Platform": '"Windows"',
},
)
context.set_default_timeout(self.settings.default_timeout_ms)
context.set_default_navigation_timeout(self.settings.default_timeout_ms)
context.add_init_script(_build_init_script())
return context

147
iaai_scraper/browser/listing.py
View File

@@ -1,147 +0,0 @@
import logging
import re
from dataclasses import asdict, dataclass, field
from typing import Any
from urllib.parse import urljoin
from playwright.sync_api import Page
from .pace import HumanPacer
from ..core.config import Settings
from ..core.utils import first_non_empty
logger = logging.getLogger("iaai_scraper.listing")
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
@dataclass(slots=True)
class ListingVehicleLink:
href: str
title: str = ""
lot_number: str | None = None
@dataclass(slots=True)
class ListingPageResult:
source_url: str
page_number: int
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
pagination_available: bool = False
next_page_detected: bool = False
class ListingCollector:
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
self.settings = settings
self.pacer = pacer
def open_cars_listing(self, page: Page) -> None:
logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
try:
page.wait_for_load_state("networkidle", timeout=15000)
except Exception:
logger.debug("networkidle timeout on listing page, continuing with current state")
try:
page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
except Exception:
logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
self.pacer.after_listing_open()
def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
applied = {"make": None, "model": None}
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
applied["make"] = make
self.pacer.after_filter_action()
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
applied["model"] = model
self.pacer.after_filter_action()
return applied
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
anchors = page.locator("a[href*='/VehicleDetail/']")
total = min(anchors.count(), self.settings.listing.page_link_limit)
links: list[ListingVehicleLink] = []
seen: set[str] = set()
for idx in range(total):
anchor = anchors.nth(idx)
href = anchor.get_attribute("href") or ""
match = VEHICLE_HREF_RE.search(href)
if not match:
continue
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
seen.add(absolute)
title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
next_page_detected = self._has_next_page(page)
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
def go_to_next_page(self, page: Page) -> bool:
selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
for selector in selectors:
locator = page.locator(selector).first
if locator.count() == 0:
continue
disabled = (locator.get_attribute("disabled") or "").lower()
aria_disabled = (locator.get_attribute("aria-disabled") or "").lower()
classes = (locator.get_attribute("class") or "").lower()
if disabled or aria_disabled == "true" or "disabled" in classes:
continue
self.pacer.move_mouse_to(page, locator)
locator.click()
try:
page.wait_for_load_state("networkidle", timeout=15000)
except Exception:
pass
self.pacer.after_page_change()
return True
return False
def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]:
self.open_cars_listing(page)
applied_filters = self.apply_filters(page, make=make, model=model)
pages: list[dict[str, object]] = []
all_links: list[str] = []
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
page_result = self.collect_current_page(page, page_number=page_number)
pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
for item in page_result.vehicle_links:
if item.href not in all_links:
all_links.append(item.href)
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
break
if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
break
if not self.go_to_next_page(page):
break
return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
@staticmethod
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
for selector in selectors:
locator = page.locator(selector).first
if locator.count() == 0:
continue
try:
locator.click()
locator.fill(value)
page.keyboard.press("Enter")
try:
page.wait_for_load_state("networkidle", timeout=15000)
except Exception:
pass
return True
except Exception:
continue
return False
@staticmethod
def _has_next_page(page: Page) -> bool:
for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
if page.locator(selector).count() > 0:
return True
return False

122
iaai_scraper/browser/network.py
View File

@@ -1,122 +0,0 @@
import json
import logging
from dataclasses import dataclass, field
from typing import Any
from urllib.parse import urlparse
from playwright.sync_api import Page, Request, Response
from ..core.config import Settings
logger = logging.getLogger("iaai_scraper.network")
@dataclass
class NetworkCapture:
# XHR/fetch перехватчик.
settings: Settings
requests: list[dict[str, Any]] = field(default_factory=list)
json_responses: list[dict[str, Any]] = field(default_factory=list)
_seen_req: set[str] = field(default_factory=set)
_seen_resp: set[str] = field(default_factory=set)
_origin: str | None = None
def attach(self, page: Page, origin_url: str | None = None) -> None:
# Подписка на сетевые события
try:
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
except Exception:
self._origin = None
page.on("request", self._on_request)
page.on("response", self._on_response)
def _is_same_origin(self, url: str) -> bool:
# Фильтр по домену
if not self.settings.capture.capture_same_origin_only or not self._origin:
return True
netloc = urlparse(url).netloc.lower()
return netloc == self._origin or netloc.endswith(".iaai.com")
def _on_request(self, request: Request) -> None:
# Берём только xhr/fetch
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(request.url):
return
if len(self.requests) >= self.settings.capture.max_requests:
return
key = f"{request.method}:{request.url}:{request.post_data or ''}"
# Убираем дубли
if key in self._seen_req:
return
self._seen_req.add(key)
self.requests.append({
"url": request.url, "method": request.method,
"resource_type": request.resource_type, "post_data": request.post_data,
})
def _on_response(self, response: Response) -> None:
# Сохраняем JSON
request = response.request
if request.resource_type not in {"xhr", "fetch"}:
return
if not self._is_same_origin(response.url):
return
if len(self.json_responses) >= self.settings.capture.max_json_responses:
return
if not self._is_json(response):
return
key = f"{request.method}:{response.url}:{response.status}"
if key in self._seen_resp:
return
self._seen_resp.add(key)
try:
payload = response.json()
except Exception:
try:
payload = json.loads(response.text())
except Exception:
return
self.json_responses.append({
"url": response.url, "status": response.status,
"request_method": request.method, "post_data": request.post_data,
"resource_type": request.resource_type, "payload": payload,
"category": self._categorize(response.url),
})
@staticmethod
def _is_json(response: Response) -> bool:
ct = (response.headers.get("content-type") or "").lower()
if "application/json" in ct or "+json" in ct:
return True
url = response.url.lower()
return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
@staticmethod
def _categorize(url: str) -> str:
# Простая категория URL
low = url.lower()
mapping = {
"images": ["image", "media", "photos", "gallery"],
"bids": ["bid", "offer", "buy-now", "buynow"],
"auction": ["auction", "sale", "lane", "branch"],
"vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
"documents": ["title", "document", "report"],
}
for cat, markers in mapping.items():
if any(m in low for m in markers):
return cat
return "other"
def export(self) -> dict[str, Any]:
responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
return {
"requests": self.requests,
"json_responses": responses,
"capture_limits": {
"same_origin_only": self.settings.capture.capture_same_origin_only,
"max_requests": self.settings.capture.max_requests,
"max_json_responses": self.settings.capture.max_json_responses,
},
}

46
iaai_scraper/browser/pace.py
View File

@@ -1,46 +0,0 @@
import random
import time
from playwright.sync_api import Locator, Page
from ..core.config import Settings
class HumanPacer:
# Random паузы между действиями.
def __init__(self, settings: Settings) -> None:
self.settings = settings
def pause(self, min_s: float, max_s: float) -> None:
if self.settings.pace.enabled:
time.sleep(random.uniform(min_s, max_s))
def after_listing_open(self) -> None:
self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
def after_filter_action(self) -> None:
self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
def before_vehicle_open(self) -> None:
self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
def after_vehicle_open(self) -> None:
self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
def between_vehicles(self) -> None:
self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
def after_page_change(self) -> None:
self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
def move_mouse_to(self, page: Page, locator: Locator) -> None:
try:
box = locator.bounding_box()
except Exception:
box = None
if not box:
return
x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
page.mouse.move(x, y, steps=random.randint(8, 18))

82
iaai_scraper/cli.py
View File

@@ -1,82 +0,0 @@
import argparse
from pathlib import Path
from .core.config import Settings
from .core.utils import save_to_json
from .scraper import IAAIScraper
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="IAAI scraper CLI")
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
subparsers = parser.add_subparsers(dest="command", required=True)
default_output_dir = Path("artifacts/json")
subparsers.add_parser("init-db", help="Create DB tables")
listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
listing_parser.add_argument("--make", default=None)
listing_parser.add_argument("--model", default=None)
listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
scrape_parser.add_argument("vehicle_url")
scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
sync_vehicle_parser.add_argument("vehicle_url")
sync_vehicle_parser.add_argument("--lane", default="iaai")
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
sync_listing_parser.add_argument("--make", default=None)
sync_listing_parser.add_argument("--model", default=None)
sync_listing_parser.add_argument("--lane", default="iaai_cars")
sync_listing_parser.add_argument("--limit", type=int, default=None)
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
return parser
def main() -> None:
parser = build_parser()
args = parser.parse_args()
runtime_settings: Settings | None = None
if args.headless is not None or args.debug:
runtime_settings = Settings()
if args.headless is not None:
runtime_settings.headless = args.headless == "true"
if args.debug:
runtime_settings.log_level = "DEBUG"
with IAAIScraper(runtime_settings) as scraper:
if args.command == "init-db":
data = scraper.init_db()
print(f"DB initialized: {data}")
return
elif args.command == "collect-listing":
data = scraper.collect_listing(make=args.make, model=args.model)
elif args.command == "scrape-vehicle":
data = scraper.scrape_vehicle_detail(args.vehicle_url)
elif args.command == "sync-vehicle":
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
else:
only_new = None if args.only_new is None else args.only_new == "true"
data = scraper.sync_listing(
make=args.make,
model=args.model,
lane=args.lane,
limit=args.limit,
only_new=only_new,
)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
if __name__ == "__main__":
main()

214
iaai_scraper/core/config.py
View File

@@ -1,214 +0,0 @@
import os
from dataclasses import dataclass, field
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
TRUE_VALUES = {"1", "true", "yes", "on"}
# Хелперы для чтения env-переменных с приведением типов
def _env_str(name: str, default: str) -> str:
value = os.getenv(name)
return value if value is not None else default
def _env_optional_str(name: str) -> str | None:
value = os.getenv(name)
if value is None:
return None
value = value.strip()
return value or None
def _env_path_str(name: str) -> str | None:
value = _env_optional_str(name)
if value is None:
return None
return str(Path(value).expanduser())
def _env_bool(name: str, default: bool) -> bool:
fallback = "true" if default else "false"
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
def _env_int(name: str, default: int) -> int:
return int(_env_str(name, str(default)).strip())
def _env_float(name: str, default: float) -> float:
return float(_env_str(name, str(default)).strip())
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
@dataclass(slots=True)
class FingerprintConfig:
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/135.0.0.0 Safari/537.36"
)
viewport_presets: tuple[dict[str, int], ...] = (
{"width": 1920, "height": 1080},
{"width": 1600, "height": 900},
{"width": 1536, "height": 864},
{"width": 1440, "height": 900},
{"width": 1366, "height": 768},
)
timezone_candidates: tuple[str, ...] = (
"America/New_York",
"America/Chicago",
"America/Los_Angeles",
)
locale: str = "en-US"
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
@dataclass(slots=True)
class CaptureConfig:
capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
# Конфиг пауз между действиями (имитация человека)
@dataclass(slots=True)
class HumanPaceConfig:
enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 2.5)
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 4.5)
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 2.0)
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 4.0)
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.5)
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 1.5)
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.3)
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 1.0)
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.5)
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 1.5)
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 3.0)
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 6.0)
# Конфиг сбора листинга (URL, лимиты страниц и машин)
@dataclass(slots=True)
class ListingConfig:
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 5)
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 100)
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 200)
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
@dataclass(slots=True)
class DatabaseConfig:
url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
# --- Конфиг Redis (URL для Celery broker) ---
@dataclass(slots=True)
class RedisConfig:
url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@dataclass(slots=True)
class CeleryConfig:
broker_url: str = _env_str("CELERY_BROKER_URL", "")
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 600)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 900)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 1)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 20)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int = _env_int("CELERY_BEAT_SYNC_LIMIT", 26)
# --- Конфиг прокси (server, username, password) ---
@dataclass(slots=True)
class ProxyConfig:
server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
@property
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
if self.username:
result["username"] = self.username
if self.password:
result["password"] = self.password
return result
# --- Конфиг сессии браузера (куки, storage_state) ---
@dataclass(slots=True)
class SessionConfig:
storage_state_path: str | None = _env_optional_str("IAAI_STORAGE_STATE_PATH")
max_age_days: int = _env_int("IAAI_SESSION_MAX_AGE_DAYS", 30)
save_on_exit: bool = _env_bool("IAAI_SESSION_SAVE_ON_EXIT", True)
@property
def enabled(self) -> bool:
return bool(self.storage_state_path)
# --- Главный объект настроек: собирает все блоки конфигурации ---
@dataclass(slots=True)
class Settings:
home_url: str = "https://www.iaai.com/"
default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 800)
max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
headless: bool = _env_bool("IAAI_HEADLESS", True)
log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", True)
raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
capture: CaptureConfig = field(default_factory=CaptureConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
session: SessionConfig = field(default_factory=SessionConfig)
# Глобальный синглтон — используется по умолчанию во всех модулях.
settings = Settings()

10
iaai_scraper/core/exceptions.py
View File

@@ -1,10 +0,0 @@
class ScraperError(Exception):
"""Base scraper exception."""
class AntiBotDetectedError(ScraperError):
"""Raised when the target website appears to block automation."""
class SiteStructureChangedError(ScraperError):
"""Raised when the page shape changed and required data is missing."""

53
iaai_scraper/core/retry.py
View File

@@ -1,53 +0,0 @@
import logging
import random
import time
from collections.abc import Callable
from functools import wraps
from typing import Any
from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
from .exceptions import AntiBotDetectedError
logger = logging.getLogger("iaai_scraper.retry")
# Типы исключений, при которых retry имеет смысл.
RETRYABLE_EXCEPTIONS = (
PlaywrightTimeoutError,
Error,
ConnectionError,
OSError,
TimeoutError,
AntiBotDetectedError,
)
def retryable(
max_attempts: int,
delay_seconds: float = 2.5,
backoff_multiplier: float = 2.0,
jitter_seconds: float = 0.0,
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
@wraps(func)
def wrapper(*args: Any, **kwargs: Any) -> Any:
last_error: Exception | None = None
for attempt in range(1, max_attempts + 1):
try:
return func(*args, **kwargs)
except RETRYABLE_EXCEPTIONS as exc:
last_error = exc
logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
if attempt < max_attempts:
sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
if jitter_seconds > 0:
sleep_for += random.uniform(0, jitter_seconds)
logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
time.sleep(sleep_for)
if last_error is not None:
raise last_error
raise RuntimeError("Retry wrapper failed without a captured exception")
return wrapper
return decorator

301
iaai_scraper/core/runtime_config.py
View File

@@ -1,301 +0,0 @@
import json
import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
logger = logging.getLogger("iaai_scraper.runtime_config")
def _normalize_text(value: str) -> str:
return value.strip().casefold()
def _text_tuple(values: Any) -> tuple[str, ...]:
return tuple(
str(item).strip()
for item in (values or [])
if str(item).strip()
)
def _int_tuple(values: Any) -> tuple[int, ...]:
result: list[int] = []
for value in values or []:
try:
result.append(int(value))
except (TypeError, ValueError):
continue
return tuple(result)
def _optional_int(value: Any) -> int | None:
if value in (None, ""):
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _optional_bool(value: Any) -> bool | None:
if value is None:
return None
if isinstance(value, bool):
return value
if isinstance(value, str):
normalized = value.strip().casefold()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return None
@dataclass(slots=True)
class RuntimeSyncConfig:
name: str | None = None
ids_initial_size: int | None = None
ids_next_size: int | None = None
ids_max_pages: int | None = None
condition_check_enabled: bool | None = None
lane: str | None = None
only_new: bool | None = None
limit: int | None = None
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig":
data = data or {}
name = str(data.get("name")).strip() if data.get("name") else None
lane = str(data.get("lane")).strip() if data.get("lane") else None
return cls(
name=name or None,
ids_initial_size=_optional_int(data.get("ids_initial_size")),
ids_next_size=_optional_int(data.get("ids_next_size")),
ids_max_pages=_optional_int(data.get("ids_max_pages")),
condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
lane=lane or None,
only_new=_optional_bool(data.get("only_new")),
limit=_optional_int(data.get("limit")),
)
@dataclass(slots=True)
class RuntimeListingConfig:
make: str | None = None
model: str | None = None
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig":
data = data or {}
make = str(data.get("make")).strip() if data.get("make") else None
model = str(data.get("model")).strip() if data.get("model") else None
return cls(make=make or None, model=model or None)
@dataclass(slots=True)
class RuntimeFieldFilters:
brands: tuple[str, ...] = ()
models: tuple[str, ...] = ()
years: tuple[int, ...] = ()
body_types: tuple[str, ...] = ()
colors: tuple[str, ...] = ()
drives: tuple[str, ...] = ()
gearboxes: tuple[str, ...] = ()
locations: tuple[str, ...] = ()
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters":
data = data or {}
return cls(
brands=_text_tuple(data.get("brands")),
models=_text_tuple(data.get("models")),
years=_int_tuple(data.get("years")),
body_types=_text_tuple(data.get("body_types")),
colors=_text_tuple(data.get("colors")),
drives=_text_tuple(data.get("drives")),
gearboxes=_text_tuple(data.get("gearboxes")),
locations=_text_tuple(data.get("locations")),
)
def is_empty(self) -> bool:
return not any([
self.brands,
self.models,
self.years,
self.body_types,
self.colors,
self.drives,
self.gearboxes,
self.locations,
])
def matches(self, values: dict[str, Any]) -> bool:
return all([
self._match_text(self.brands, values.get("brand")),
self._match_text(self.models, values.get("model")),
self._match_int(self.years, values.get("year")),
self._match_text(self.body_types, values.get("body_type")),
self._match_text(self.colors, values.get("color")),
self._match_text(self.drives, values.get("drive")),
self._match_text(self.gearboxes, values.get("gearbox")),
self._match_text(self.locations, values.get("location")),
])
@staticmethod
def _match_text(allowed: tuple[str, ...], value: Any) -> bool:
if not allowed:
return True
normalized = _normalize_text(str(value or ""))
return normalized in {_normalize_text(item) for item in allowed}
@staticmethod
def _match_int(allowed: tuple[int, ...], value: Any) -> bool:
if not allowed:
return True
parsed = _optional_int(value)
return parsed in set(allowed)
@dataclass(slots=True)
class RuntimeRangeFilter:
min: int | None = None
max: int | None = None
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter":
data = data or {}
return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max")))
def is_empty(self) -> bool:
return self.min is None and self.max is None
def matches(self, value: Any) -> bool:
parsed = _optional_int(value)
if parsed is None:
return self.is_empty()
if self.min is not None and parsed < self.min:
return False
if self.max is not None and parsed > self.max:
return False
return True
@dataclass(slots=True)
class RuntimeFlagFilters:
damaged_only: bool | None = None
run_and_drive: bool | None = None
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters":
data = data or {}
return cls(
damaged_only=_optional_bool(data.get("damaged_only")),
run_and_drive=_optional_bool(data.get("run_and_drive")),
)
def is_empty(self) -> bool:
return self.damaged_only is None and self.run_and_drive is None
def matches(self, values: dict[str, Any]) -> bool:
if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only:
return False
if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive:
return False
return True
@dataclass(slots=True)
class RuntimeFiltersConfig:
include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters)
@classmethod
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig":
data = data or {}
legacy_fields = RuntimeFieldFilters.from_dict(data)
include_payload = data.get("include")
exclude_payload = data.get("exclude")
flat_exclude_payload = {
"brands": data.get("exclude_brands"),
"models": data.get("exclude_models"),
"years": data.get("exclude_years"),
"body_types": data.get("exclude_body_types"),
"colors": data.get("exclude_colors"),
"drives": data.get("exclude_drives"),
"gearboxes": data.get("exclude_gearboxes"),
"locations": data.get("exclude_locations"),
}
include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields
exclude = (
RuntimeFieldFilters.from_dict(exclude_payload)
if exclude_payload is not None
else RuntimeFieldFilters.from_dict(flat_exclude_payload)
)
return cls(
include=include,
exclude=exclude,
price=RuntimeRangeFilter.from_dict(data.get("price")),
mileage=RuntimeRangeFilter.from_dict(data.get("mileage")),
flags=RuntimeFlagFilters.from_dict(data.get("flags")),
)
def is_empty(self) -> bool:
return all([
self.include.is_empty(),
self.exclude.is_empty(),
self.price.is_empty(),
self.mileage.is_empty(),
self.flags.is_empty(),
])
def matches(self, values: dict[str, Any]) -> bool:
if not self.include.matches(values):
return False
if not self._matches_exclude(values):
return False
if not self.price.matches(values.get("price")):
return False
if not self.mileage.matches(values.get("mileage")):
return False
if not self.flags.matches(values):
return False
return True
def _matches_exclude(self, values: dict[str, Any]) -> bool:
if self.exclude.is_empty():
return True
return not self.exclude.matches(values)
@dataclass(slots=True)
class RuntimeConfig:
sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
@classmethod
def from_file(cls, config_path: str | None) -> "RuntimeConfig":
if not config_path:
return cls()
path = Path(config_path)
if not path.exists():
logger.info("Runtime config file not found: %s", path)
return cls()
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception as exc:
logger.warning("Failed to read runtime config %s: %s", path, exc)
return cls()
return cls(
sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
listing=RuntimeListingConfig.from_dict(payload.get("listing")),
filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
)

360
iaai_scraper/parsing/mapper.py
View File

@@ -1,360 +0,0 @@
import re
from datetime import datetime, timezone
from typing import Any
from urllib.parse import urlparse
from ..core.utils import first_non_empty
from ..storage.enums import (
BODY_TYPE_ENUM_VALUES,
COUNTRY_ENUM_VALUES,
CURRENCY_ENUM_VALUES,
DRIVE_ENUM_VALUES,
GEARBOX_ENUM_VALUES,
ORIGIN_ENUM_VALUES,
SELLING_TYPE_ENUM_VALUES,
STEERING_WHEEL_ENUM_VALUES,
)
from ..storage.schemas import CarRecord, ImageRecord
class CarMapper:
# IAAI data → CarRecord.
BODY_MAP = {
"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
}
DRIVE_MAP = {
"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
}
GEARBOX_MAP = {
"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
}
STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
COUNTRY_MAP = {
"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
}
NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
# Собираем нормализованную DB-модель.
vehicle_summary = vehicle_summary or {}
payload_insights = payload_insights or {}
core = payload_insights.get("vehicle_core", {})
pricing = payload_insights.get("pricing", {})
damage = payload_insights.get("damage", {})
auction = payload_insights.get("auction", {})
images = payload_insights.get("images", {})
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
parser_id = f"iaai:{origin_id}"
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
price = self._first_parsed_int(
[
pricing.get("buy_now"),
pricing.get("current_bid"),
vehicle_summary.get("buy_now"),
vehicle_summary.get("current_bid"),
pricing.get("actual_cash_value"),
],
self._to_money_int,
)
mileage = self._first_parsed_int(
[core.get("odometer"), vehicle_summary.get("odometer"), 0],
self._to_int,
) or 0
color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
is_damaged = self._bool_damage(damage, vehicle_summary)
is_sold = self._bool_sold(auction)
one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
currency = self._normalize_currency(
first_non_empty(
[
pricing.get("currency"),
vehicle_summary.get("currency"),
pricing.get("buy_now"),
pricing.get("current_bid"),
vehicle_summary.get("buy_now"),
vehicle_summary.get("current_bid"),
"USD",
]
)
)
slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id])))
images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
origin = "IAAI"
return CarRecord(
parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
)
@staticmethod
def _as_str(value: Any) -> str:
return "" if value is None else str(value).strip()
@staticmethod
def _to_int(value: Any) -> int | None:
if value is None:
return None
if isinstance(value, bool):
return int(value)
if isinstance(value, (int, float)):
return int(value)
digits = re.sub(r"[^\d]", "", str(value))
return int(digits) if digits else None
@classmethod
def _to_money_int(cls, value: Any) -> int | None:
# Нормализация стоимости из разных форматов.
if value is None:
return None
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
return int(value)
text = str(value).strip()
if not text:
return None
lowered = text.lower()
if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
return None
numbers = re.findall(r"\d[\d\s.,]*", text)
if not numbers:
return None
best: int | None = None
for number in numbers:
clean = number.replace(" ", "")
if "," in clean and "." in clean:
# Поддержка 1,234.56 и 1.234,56.
if clean.rfind(",") > clean.rfind("."):
clean = clean.replace(".", "").replace(",", ".")
else:
clean = clean.replace(",", "")
elif "," in clean:
parts = clean.split(",")
# 123,45 -> 123.45, иначе разделитель тысяч.
if len(parts[-1]) in {1, 2} and len(parts) == 2:
clean = clean.replace(",", ".")
else:
clean = clean.replace(",", "")
elif "." in clean:
parts = clean.split(".")
if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
clean = clean.replace(".", "")
try:
parsed = int(float(clean))
except ValueError:
continue
if parsed > 0 and (best is None or parsed > best):
best = parsed
return best
@staticmethod
def _first_parsed_int(values: list[Any], parser) -> int | None:
for value in values:
parsed = parser(value)
if parsed is not None:
return parsed
return None
@staticmethod
def _to_year(value: Any) -> int | None:
parsed = CarMapper._to_int(value)
if parsed is None:
return None
if 1900 <= parsed <= 2100:
return parsed
return None
def _to_engine_cc(self, value: Any) -> int | None:
# Поддержка литров и cc.
text = str(value).lower().strip() if value is not None else ""
if not text:
return None
if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
return int(float(liters_match.group(1)) * 1000)
if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
return int(cubic_match.group(1))
return int(text) if re.match(r"^\d+$", text) else None
def _normalize_currency(self, value: Any) -> str:
text = self._as_str(value)
upper = text.upper() or "USD"
if any(token in text for token in ["€", "EUR"]):
return "EUR"
if any(token in text for token in ["¥", "JPY"]):
return "JPY"
if any(token in text for token in ["₩", "KRW"]):
return "KRW"
if any(token in text for token in ["£", "GBP"]):
return "GBP"
if any(token in text for token in ["₽", "RUB"]):
return "RUB"
if any(token in text for token in ["AED", "د.إ"]):
return "AED"
if any(token in text for token in ["CA$", "CAD"]):
return "CAD"
text = upper
if text in CURRENCY_ENUM_VALUES:
return text
return "USD" if "$" in str(value) else "USD"
def _normalize_drive(self, value: Any) -> str | None:
return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
def _normalize_gearbox(self, value: Any) -> str | None:
return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
def _normalize_steering(self, value: Any) -> str | None:
return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
def _normalize_body_type(self, value: Any) -> str:
return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
def _normalize_country(self, value: Any) -> str:
fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
def _normalize_selling_type(self, value: Any) -> str:
text = self._as_str(value) or "AUCTION"
return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
def _map_value(
self,
value: Any,
mapping: dict[str, str],
allowed_values: tuple[str, ...],
*,
empty_default: str | None,
fallback: str | None,
) -> str | None:
# Общий helper для enum-нормализации.
text = self._as_str(value).lower()
if not text:
return empty_default
if (mapped := mapping.get(text)) and mapped in allowed_values:
return mapped
for marker, mapped in mapping.items():
if marker in text and mapped in allowed_values:
return mapped
return fallback
@staticmethod
def _normalize_color(value: Any) -> str:
text = str(value).strip() if value is not None else "other"
if not text:
return "other"
if "/" in text:
text = text.split("/")[0].strip()
return text.lower() or "other"
@staticmethod
def _boolish(value: Any) -> bool:
return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
text = self._as_str(value).lower()
if text and text not in self.NO_DAMAGE_MARKERS:
return True
return False
def _bool_sold(self, auction: dict[str, Any]) -> bool:
return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
# Для imageKeys берем самый большой размер.
best_by_key: dict[str, str] = {}
key_order: list[str] = []
non_keyed: list[str] = []
for item in urls:
if not isinstance(item, str):
continue
url = item.strip()
if not url:
continue
if m := re.search(r'imageKeys=([^&]+)', url):
img_key = m.group(1)
w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
existing = best_by_key.get(img_key)
if existing is None:
best_by_key[img_key] = url
key_order.append(img_key)
else:
existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
if w > existing_w:
best_by_key[img_key] = url
elif url not in non_keyed:
non_keyed.append(url)
deduped = [best_by_key[k] for k in key_order] + non_keyed
return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
@staticmethod
def _make_fullres_url(url: str) -> str:
if "vis.iaai.com" in url:
return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
return url
@staticmethod
def _make_preview_url(url: str) -> str:
if "vis.iaai.com" in url:
preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
if preview != url:
return preview
return url
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
# Берем lot_number, затем vin, затем хвост URL.
for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
text = self._as_str(value)
if text:
return text
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
if "~" in tail:
tail = tail.split("~")[0]
return tail or self._slugify(vehicle_url)
@staticmethod
def _slugify(value: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"

368
iaai_scraper/parsing/parser.py
View File

@@ -1,368 +0,0 @@
import html as html_module
import json
import logging
import re
from typing import Any
from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty
logger = logging.getLogger("iaai_scraper.parsers")
class VehicleParser:
# DOM + JSON парсер страницы авто.
SUMMARY_KEY_MAP = {
"vin": {"vin", "vehicleidentificationnumber"},
"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
"year": {"year"},
"make": {"make", "manufacturer", "brand"},
"model": {"model"},
"trim": {"trim", "series"},
"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
"secondary_damage": {"secondarydamage"},
"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
"actual_cash_value": {"actualcashvalue", "acv"},
"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
"keys": {"keys", "keystatus"},
"title": {"titletype", "title", "documenttype"},
"seller": {"seller", "sellername"},
"location": {"location", "branchname", "auctionlocation", "branch"},
"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
"body_type": {"bodytype", "bodystyle"},
"drive": {"driveline", "drive"},
"gearbox": {"transmission", "gearbox"},
"engine": {"engine", "enginevolume"},
"fuel_type": {"fueltype", "fuel"},
"cylinders": {"cylinders", "cylindercount"},
"color": {"color", "primarycolor", "exteriorcolor"},
}
DOM_LABEL_MAP: dict[str, str] = {
"stock #": "lot_number",
"vin (status)": "vin",
"vin": "vin",
"primary damage": "primary_damage",
"secondary damage": "secondary_damage",
"odometer": "odometer",
"body style": "body_type",
"engine": "engine",
"transmission": "gearbox",
"drive line type": "drive",
"fuel type": "fuel_type",
"cylinders": "cylinders",
"exterior/interior": "color",
"exterior color": "color",
"model": "model",
"series": "trim",
"selling branch": "location",
"vehicle location": "vehicle_location",
"auction date and time": "auction_date",
"lane/run #": "lane",
"actual cash value": "actual_cash_value",
"estimated repair cost": "estimated_repair_cost",
"seller": "seller",
"title/sale doc": "title",
"title/sale doc brand": "title_brand",
"start code": "run_and_drive",
"key": "keys",
"manufactured in": "manufactured_in",
"vehicle class": "vehicle_class",
}
def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
result: dict[str, str] = {}
if not dom_text:
return result
lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
known_labels = set(self.DOM_LABEL_MAP.keys())
for i, line in enumerate(lines):
clean = line.rstrip(":").lower().strip()
if clean in known_labels and i + 1 < len(lines):
value = lines[i + 1].strip()
if value.rstrip(":").lower().strip() in known_labels:
continue
if value.lower() in ("more actions", "view", "print", "share", "back to results"):
continue
field_name = self.DOM_LABEL_MAP[clean]
if field_name not in result or not result[field_name]:
result[field_name] = value
return result
def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
result: dict[str, str | None] = {"year": None, "make": None, "model": None}
title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
if title_match:
result["year"] = title_match.group(1)
result["make"] = title_match.group(2)
result["model"] = title_match.group(3)
return result
dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
if dom_match:
result["year"] = dom_match.group(1)
result["make"] = dom_match.group(2)
result["model"] = dom_match.group(3).strip()
return result
def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
page_html = page_html or ""
dom_text = dom_text or ""
network_dump = network_dump or {}
responses = network_dump.get("json_responses", [])
payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
dom_kv = self._parse_dom_key_value_pairs(dom_text)
page_title = ""
title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
if title_match:
page_title = title_match.group(1).strip()
title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
summary: dict[str, Any] = {"source_url": vehicle_url}
for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
values: list[Any] = []
for payload in payloads:
values.extend(deep_find_key(payload, candidate_keys))
if field in dom_kv:
values.append(dom_kv[field])
summary[field] = first_non_empty(values)
summary["year"] = summary.get("year") or title_parsed.get("year")
summary["make"] = summary.get("make") or title_parsed.get("make")
summary["model"] = summary.get("model") or title_parsed.get("model")
summary["trim"] = summary.get("trim") or dom_kv.get("trim")
summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html)
if summary.get("vin") and isinstance(summary["vin"], str):
vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip()
vin_match = VIN_RE.search(vin_clean)
summary["vin"] = vin_match.group(1) if vin_match else vin_clean
summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
for dom_field, dom_value in dom_kv.items():
if dom_field not in summary or not summary[dom_field]:
summary[dom_field] = dom_value
prices = self._extract_prices_from_text(dom_text)
if not summary.get("actual_cash_value") and prices:
summary["actual_cash_value"] = prices[0]
if not summary.get("buy_now"):
buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE)
if buy_now_match:
summary["buy_now"] = buy_now_match.group(1)
elif prices:
summary["buy_now"] = prices[0]
if not summary.get("current_bid") and len(prices) > 1:
summary["current_bid"] = prices[1]
embedded = self._extract_embedded_json(page_html)
for item in embedded:
p = item.get("payload")
if isinstance(p, (dict, list)):
payloads.append(p)
for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
if not summary.get(field):
val = first_non_empty(deep_find_key(p, candidate_keys))
if val:
summary[field] = val
return {
"vehicle_summary": summary,
"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url),
"embedded_json": embedded,
"dom_hints": self._dom_hints(dom_text),
"access_notes": self._build_access_notes(summary, responses),
}
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
return {
"vehicle_core": {
"vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"),
"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
"color": summary.get("color"), "keys": summary.get("keys"),
},
"pricing": {
"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
"currency": self._guess_currency(summary),
},
"bids": self._build_bid_insights(summary, payloads),
"damage": {
"primary": summary.get("primary_damage"),
"secondary": summary.get("secondary_damage"),
"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
},
"auction": {
"auction_date": summary.get("auction_date"),
"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
},
"images": {"count": len(image_urls), "urls": image_urls},
"source_endpoints": self._build_source_endpoints(responses),
}
def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
return {
"amount": summary.get("current_bid"),
"currency": self._guess_currency(summary),
"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
}
def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
for item in responses:
url = item.get("url", "")
category = item.get("category", "other")
if category == "vehicle":
mapping["vehicle"].append(url)
lowered = url.lower()
if any(token in lowered for token in ["bid", "offer"]):
mapping["bids"].append(url)
if any(token in lowered for token in ["damage", "report"]):
mapping["damage"].append(url)
if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
mapping["auction"].append(url)
elif category in mapping:
mapping[category].append(url)
return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
endpoints = [item.get("url", "") for item in responses]
dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
return {
"vin_visible": bool(summary.get("vin")),
"images_visible": bool(summary.get("image_urls")),
"network_json_count": len(responses),
"possible_captcha": bool(dom_hints.get("has_captcha_text")),
"possible_antibot": bool(dom_hints.get("has_antibot_text")),
"observed_endpoints": endpoints[:20],
}
@staticmethod
def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
lowered = {key.lower() for key in keys}
values: list[Any] = []
for payload in payloads:
values.extend(deep_find_key(payload, lowered))
return values
@staticmethod
def _guess_currency(summary: dict[str, Any]) -> str:
for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
value = str(summary.get(key) or "")
if "$" in value:
return "USD"
if "€" in value:
return "EUR"
if "¥" in value:
return "JPY"
return "USD"
@staticmethod
def _extract_vin(text: str) -> str | None:
match = VIN_RE.search(text or "")
return match.group(1) if match else None
@staticmethod
def _extract_lot_number(text: str) -> str | None:
match = LOT_RE.search(text or "")
return match.group(1) if match else None
@staticmethod
def _extract_prices_from_text(text: str) -> list[str]:
return [match.group(1) for match in PRICE_RE.finditer(text or "")]
@staticmethod
def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
extracted: list[dict[str, Any]] = []
for script_text in scripts:
if "{" not in script_text and "[" not in script_text:
continue
try:
parsed = json.loads(script_text.strip())
except Exception:
continue
extracted.append({"type": "inline_json", "payload": parsed})
return extracted
@staticmethod
def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
vehicle_key = ""
key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
if key_match:
vehicle_key = key_match.group(1)
found: list[str] = []
for payload in payloads:
found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
flat: list[str] = []
seen_flat: set[str] = set()
for item in found:
if isinstance(item, str) and item.startswith("http"):
cleaned = html_module.unescape(item)
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
continue
if cleaned not in seen_flat:
seen_flat.add(cleaned)
flat.append(cleaned)
elif isinstance(item, list):
for child in item:
if isinstance(child, str) and child.startswith("http"):
cleaned = html_module.unescape(child)
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
continue
if cleaned not in seen_flat:
seen_flat.add(cleaned)
flat.append(cleaned)
for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
for match in re.finditer(pattern, html or "", re.IGNORECASE):
url = html_module.unescape(match.group(1).strip())
if not url.startswith("http") or url in seen_flat:
continue
lowered = url.lower()
if vehicle_key and vehicle_key in url:
seen_flat.add(url)
flat.append(url)
elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
if vehicle_key and vehicle_key not in url:
continue
seen_flat.add(url)
flat.append(url)
if vehicle_key:
for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
cleaned = html_module.unescape(url)
if cleaned not in seen_flat and vehicle_key in cleaned:
seen_flat.add(cleaned)
flat.append(cleaned)
filtered: list[str] = []
for url in flat:
lowered = url.lower()
if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
continue
if "vis.iaai.com" in lowered and "/resizer" not in lowered:
continue
filtered.append(url)
return filtered
@staticmethod
def _dom_hints(text: str) -> dict[str, Any]:
lowered = (text or "").lower()
return {
"has_buy_now_text": "buy now" in lowered,
"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
"has_damage_text": "damage" in lowered,
"has_vin_text": "vin" in lowered,
"has_title_text": "title" in lowered,
"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
}

317
iaai_scraper/proxy_bridge.py
View File

@@ -1,317 +0,0 @@
from __future__ import annotations
import logging
import os
import select
import socket
import socketserver
import struct
import threading
from urllib.parse import urlsplit
BUFFER_SIZE = 65536
CRLF = b"\r\n"
DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1")
DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))
SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "")
SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "")
RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60"))
MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64"))
logger = logging.getLogger("proxy_bridge")
class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
allow_reuse_address = True
daemon_threads = True
def __init__(self, server_address, request_handler_class):
super().__init__(server_address, request_handler_class)
self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS)
def process_request_thread(self, request, client_address):
with self._worker_semaphore:
super().process_request_thread(request, client_address)
def _recv_exact(sock: socket.socket, size: int) -> bytes:
data = b""
while len(data) < size:
chunk = sock.recv(size - len(data))
if not chunk:
raise ConnectionError("Unexpected EOF from SOCKS5 server")
data += chunk
return data
def _socks5_connect(host: str, port: int) -> socket.socket:
if not SOCKS5_HOST:
raise RuntimeError("SOCKS5_PROXY_HOST is not configured")
upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30)
upstream.settimeout(30)
methods = [0x00]
if SOCKS5_USER or SOCKS5_PASS:
methods = [0x02]
upstream.sendall(bytes([0x05, len(methods), *methods]))
version, method = _recv_exact(upstream, 2)
if version != 0x05 or method == 0xFF:
upstream.close()
raise ConnectionError("SOCKS5 authentication negotiation failed")
if method == 0x02:
username = SOCKS5_USER.encode("utf-8")
password = SOCKS5_PASS.encode("utf-8")
if len(username) > 255 or len(password) > 255:
upstream.close()
raise ValueError("SOCKS5 username/password too long")
upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password)
auth_version, auth_status = _recv_exact(upstream, 2)
if auth_version != 0x01 or auth_status != 0x00:
upstream.close()
raise ConnectionError("SOCKS5 username/password authentication failed")
try:
socket.inet_aton(host)
addr_type = 0x01
addr_payload = socket.inet_aton(host)
except OSError:
host_bytes = host.encode("idna")
if len(host_bytes) > 255:
upstream.close()
raise ValueError("Target host is too long for SOCKS5 domain format")
addr_type = 0x03
addr_payload = bytes([len(host_bytes)]) + host_bytes
request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port)
upstream.sendall(request)
response_head = _recv_exact(upstream, 4)
version, reply, _reserved, reply_addr_type = response_head
if version != 0x05 or reply != 0x00:
upstream.close()
raise ConnectionError(f"SOCKS5 connect failed with code {reply}")
if reply_addr_type == 0x01:
_recv_exact(upstream, 4)
elif reply_addr_type == 0x03:
domain_len = _recv_exact(upstream, 1)[0]
_recv_exact(upstream, domain_len)
elif reply_addr_type == 0x04:
_recv_exact(upstream, 16)
_recv_exact(upstream, 2)
upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
return upstream
def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None:
sockets = [left, right]
left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
try:
while True:
readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS)
if exceptional:
break
if not readable:
logger.debug("Relay idle timeout reached; closing sockets")
return
for current in readable:
other = right if current is left else left
data = current.recv(BUFFER_SIZE)
if not data:
return
other.sendall(data)
finally:
for sock in sockets:
try:
sock.shutdown(socket.SHUT_RDWR)
except OSError:
pass
try:
sock.close()
except OSError:
pass
class ProxyHandler(socketserver.StreamRequestHandler):
def handle(self) -> None:
try:
request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip()
if not request_line:
return
method, target, version = request_line.split()
headers = self._read_headers()
logger.info("%s %s", method, target)
if method.upper() == "CONNECT":
host, port = self._parse_connect_target(target)
logger.info("CONNECT %s:%s", host, port)
upstream = _socks5_connect(host, port)
self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF)
self.wfile.flush()
_relay_bidirectional(self.connection, upstream)
return
host, port, path = self._parse_forward_target(target, headers)
upstream = _socks5_connect(host, port)
self._send_forward_request(upstream, method, path, version, headers)
body = self._read_request_body(headers)
if body:
upstream.sendall(body)
_relay_bidirectional(self.connection, upstream)
except Exception as exc:
logger.exception("Proxy bridge request failed: %s", exc)
try:
self.wfile.write(
b"HTTP/1.1 502 Bad Gateway" + CRLF
+ b"Connection: close" + CRLF
+ b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF
+ b"Bad Gateway"
)
self.wfile.flush()
except OSError:
pass
def _read_headers(self) -> list[tuple[str, str]]:
headers: list[tuple[str, str]] = []
while True:
line = self.rfile.readline(BUFFER_SIZE)
if line in {CRLF, b"\n", b""}:
break
decoded = line.decode("iso-8859-1")
if ":" not in decoded:
continue
name, value = decoded.split(":", 1)
headers.append((name.strip(), value.strip()))
return headers
@staticmethod
def _parse_connect_target(target: str) -> tuple[str, int]:
if target.startswith("["):
end = target.find("]")
if end == -1 or len(target) <= end + 2 or target[end + 1] != ":":
raise ValueError("Invalid CONNECT target")
host = target[1:end]
port_text = target[end + 2 :]
return host, int(port_text)
host, port_text = target.rsplit(":", 1)
return host, int(port_text)
@staticmethod
def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]:
if target.startswith("http://"):
parts = urlsplit(target)
port = parts.port or 80
path = parts.path or "/"
if parts.query:
path += f"?{parts.query}"
return parts.hostname or "", port, path
if target.startswith("https://"):
raise ValueError("HTTPS absolute-form request must use CONNECT")
host_header = next((value for name, value in headers if name.lower() == "host"), "")
if not host_header:
raise ValueError("Missing Host header")
if ":" in host_header:
host, port_text = host_header.rsplit(":", 1)
return host, int(port_text), target
return host_header, 80, target
def _send_forward_request(
self,
upstream: socket.socket,
method: str,
path: str,
version: str,
headers: list[tuple[str, str]],
) -> None:
filtered_headers: list[tuple[str, str]] = []
hop_by_hop = {
"proxy-connection",
"proxy-authorization",
"connection",
"keep-alive",
"te",
"trailer",
"transfer-encoding",
"upgrade",
}
for name, value in headers:
if name.lower() in hop_by_hop:
continue
filtered_headers.append((name, value))
request_head = [f"{method} {path} {version}\r\n"]
request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers)
request_head.append("\r\n")
upstream.sendall("".join(request_head).encode("iso-8859-1"))
def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes:
transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "")
if "chunked" in transfer_encoding.lower():
return self._read_chunked_request_body()
content_length = next((value for name, value in headers if name.lower() == "content-length"), None)
if not content_length:
return b""
return self.rfile.read(int(content_length))
def _read_chunked_request_body(self) -> bytes:
chunks: list[bytes] = []
while True:
size_line = self.rfile.readline(BUFFER_SIZE)
if not size_line:
raise ConnectionError("Unexpected EOF in chunked request")
size_text = size_line.strip().split(b";", 1)[0]
chunk_size = int(size_text, 16)
chunks.append(size_line)
if chunk_size == 0:
while True:
trailer_line = self.rfile.readline(BUFFER_SIZE)
if not trailer_line:
raise ConnectionError("Unexpected EOF in chunked trailers")
chunks.append(trailer_line)
if trailer_line in {CRLF, b"\n"}:
return b"".join(chunks)
chunk_data = self.rfile.read(chunk_size)
if len(chunk_data) != chunk_size:
raise ConnectionError("Unexpected EOF in chunk body")
chunks.append(chunk_data)
chunk_end = self.rfile.read(2)
if chunk_end != CRLF:
raise ConnectionError("Invalid chunk terminator")
chunks.append(chunk_end)
def main() -> None:
if not SOCKS5_HOST:
raise SystemExit("SOCKS5_PROXY_HOST is required")
logging.basicConfig(
level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(),
format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s",
)
with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server:
logger.info(
"Listening on %s:%s -> socks5://%s:%s (max_workers=%s)",
DEFAULT_LISTEN_HOST,
DEFAULT_LISTEN_PORT,
SOCKS5_HOST,
SOCKS5_PORT,
MAX_WORKERS,
)
server.serve_forever()
if __name__ == "__main__":
main()

506
iaai_scraper/scraper.py
View File

@@ -1,506 +0,0 @@
import json
import logging
import os
import re
import signal
import time
import uuid
from datetime import datetime, timezone
from pathlib import Path
from playwright.sync_api import Error as PlaywrightError
from playwright.sync_api import BrowserContext, Page, sync_playwright
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
from .browser import BrowserFactory, HumanPacer, NetworkCapture
from .core.config import Settings, settings
from .core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from .core.logs import set_trace_id, setup_logging
from .core.retry import retryable
from .core.runtime_config import RuntimeConfig
from .core.utils import save_to_json
from .parsing.mapper import CarMapper
from .parsing.parser import VehicleParser
from .storage.db import PersistenceService
from .browser.listing import ListingCollector
from .storage.schemas import CarRecord
logger = logging.getLogger("iaai_scraper.scraper")
VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
class IAAIScraper:
@staticmethod
def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None:
dom_hints = parsed.get("dom_hints", {}) or {}
access_notes = parsed.get("access_notes", {}) or {}
summary = parsed.get("vehicle_summary", {}) or {}
if dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text"):
raise AntiBotDetectedError(f"IAAI anti-bot detected for {vehicle_url}")
if access_notes.get("possible_captcha") or access_notes.get("possible_antibot"):
raise AntiBotDetectedError(f"IAAI blocked or challenged request for {vehicle_url}")
has_identity = bool(summary.get("lot_number") or summary.get("vin") or summary.get("make") or summary.get("model"))
if not has_identity:
raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}")
@staticmethod
def _extract_origin_id_from_url(vehicle_url: str) -> str | None:
match = VEHICLE_ID_RE.search(vehicle_url)
if not match:
return None
return match.group(1)
def __init__(self, runtime_settings: Settings | None = None) -> None:
self.settings = runtime_settings or settings
setup_logging(self.settings.log_level, self.settings.log_file)
self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file)
self.trace_id = uuid.uuid4().hex[:12]
set_trace_id(self.trace_id)
self.playwright = None
self.browser = None
self.context: BrowserContext | None = None
self.browser_factory = BrowserFactory(self.settings)
self.pacer = HumanPacer(self.settings)
self.listing_collector = ListingCollector(self.settings, self.pacer)
self.vehicle_parser = VehicleParser()
self.car_mapper = CarMapper()
self.persistence = PersistenceService(self.settings)
self._shutdown_requested = False
def _new_trace_id(self, prefix: str) -> str:
trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}"
self.trace_id = trace_id
set_trace_id(trace_id)
return trace_id
def __enter__(self) -> "IAAIScraper":
if self.playwright is None:
self.playwright = sync_playwright().start()
if self.browser is None:
self.browser = self.browser_factory.create_browser(self.playwright)
return self
def __exit__(self, exc_type, exc, tb) -> None:
self.close()
def close(self) -> None:
if self.context is not None:
try:
self._save_storage_state()
self.context.close()
except PlaywrightError:
pass
finally:
self.context = None
if self.browser is not None:
try:
self.browser.close()
except PlaywrightError:
pass
finally:
self.browser = None
if self.playwright is not None:
try:
self.playwright.stop()
except PlaywrightError:
pass
finally:
self.playwright = None
def _load_storage_state(self) -> str | None:
"""Load saved browser session (cookies + localStorage) if valid."""
session_cfg = self.settings.session
if not session_cfg.enabled:
return None
state_path = session_cfg.storage_state_path
if not state_path or not os.path.isfile(state_path):
logger.info("No saved session found at %s", state_path)
return None
try:
mtime = os.path.getmtime(state_path)
age_days = (time.time() - mtime) / 86400
if age_days > session_cfg.max_age_days:
logger.info("Session expired (%.1f days old, max %d). Starting fresh.", age_days, session_cfg.max_age_days)
os.remove(state_path)
return None
logger.info("Reusing saved session from %s (%.1f days old)", state_path, age_days)
return state_path
except Exception as exc:
logger.warning("Failed to load session state: %s", exc)
return None
def _save_storage_state(self) -> None:
"""Save browser session (cookies + localStorage) to disk."""
session_cfg = self.settings.session
if not session_cfg.enabled or not session_cfg.save_on_exit:
return
if not self.context:
return
state_path = session_cfg.storage_state_path
try:
Path(state_path).parent.mkdir(parents=True, exist_ok=True)
self.context.storage_state(path=state_path)
logger.info("Session state saved to %s", state_path)
except Exception as exc:
logger.warning("Failed to save session state: %s", exc)
def _new_context(self, storage_state: str | None = None) -> BrowserContext:
if self.browser is None:
self.__enter__()
if self.context:
self._save_storage_state()
self.context.close()
effective_state = storage_state or self._load_storage_state()
self.context = self.browser_factory.create_context(self.browser, storage_state=effective_state)
return self.context
def init_db(self):
self.persistence.create_tables()
return {"status": "ok", "database_url": self.settings.database.url}
def _get_unauthenticated_page(self) -> Page:
context = self._new_context()
return context.new_page()
def collect_listing(self, make: str | None = None, model: str | None = None):
page = self._get_unauthenticated_page()
try:
listing = self.listing_collector.collect_listing_links(page, make=make, model=model)
finally:
page.close()
return {
"status": "ok",
**listing,
}
def _get_page(self) -> Page:
if not self.context:
self._new_context()
return self.context.new_page()
@retryable(max_attempts=3, jitter_seconds=0.25)
def scrape_vehicle_detail(self, vehicle_url: str):
# Открыть страницу, перехватить JSON, вернуть данные.
page = self._get_page()
try:
return self._scrape_on_page(page, vehicle_url)
finally:
page.close()
def _scrape_on_page(self, page: Page, vehicle_url: str):
trace_id = self._new_trace_id("scrape")
started_at = time.perf_counter()
capture = NetworkCapture(self.settings)
capture.attach(page, origin_url=vehicle_url)
page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000)
try:
page.wait_for_load_state("networkidle", timeout=10000)
except PlaywrightTimeoutError:
pass
time.sleep(self.settings.network_settle_ms / 1000)
html = page.content()
try:
dom_text = page.locator("body").inner_text(timeout=10_000)
except Exception:
dom_text = ""
network_dump = capture.export()
parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump)
self._raise_if_blocked_or_incomplete(parsed, vehicle_url)
db_record = self.car_mapper.map_to_car_record(
vehicle_url=vehicle_url,
vehicle_summary=parsed.get("vehicle_summary", {}),
payload_insights=parsed.get("payload_insights", {}),
)
result = {
"trace_id": trace_id,
"source_url": vehicle_url,
"fetched_at_epoch": int(time.time()),
"elapsed_seconds": round(time.perf_counter() - started_at, 3),
"network": network_dump,
**parsed,
"db_record": db_record.model_dump(mode="json"),
}
if self.settings.raw_output_json:
save_to_json(network_dump, self.settings.raw_output_json)
return result
def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"):
# Scrape + upsert одного авто.
trace_id = self._new_trace_id("sync-vehicle")
started_at = time.perf_counter()
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane=lane)
ids_fetched = 1
cars_upserted = 0
cars_failed = 0
images_upserted = 0
status = "failed"
error_summary = None
try:
scrape_result = self.scrape_vehicle_detail(vehicle_url)
db_record = scrape_result.get("db_record")
if not db_record:
raise RuntimeError("Scrape result does not contain db_record")
record = CarRecord.model_validate(db_record)
upsert = self.persistence.upsert_car(record)
cars_upserted = 1
images_upserted = int(upsert.get("images_upserted", 0))
status = "success"
return {
"trace_id": trace_id,
"status": status,
"run_id": run_id,
"vehicle_url": vehicle_url,
"db_action": upsert.get("action"),
"images_upserted": images_upserted,
"elapsed_seconds": round(time.perf_counter() - started_at, 3),
"db_record": db_record,
}
except Exception as exc:
cars_failed = 1
status = "failed"
error_summary = str(exc)
raise
finally:
self.persistence.finish_sync_run(
run_id,
status=status,
ids_fetched=ids_fetched,
cars_upserted=cars_upserted,
cars_failed=cars_failed,
images_upserted=images_upserted,
error_summary=error_summary,
)
def sync_listing(
self,
make: str | None = None,
model: str | None = None,
lane: str = "iaai_cars",
limit: int | None = None,
only_new: bool | None = None,
):
# Листинг + sync всех найденных машин.
# Применяем runtime_config как дефолты (CLI/API аргументы имеют приоритет).
rc = self.runtime_config.sync
if limit is None and rc.limit is not None:
limit = rc.limit
if only_new is None and rc.only_new is not None:
only_new = rc.only_new
if lane == "iaai_cars" and rc.lane is not None:
lane = rc.lane
trace_id = self._new_trace_id("sync-listing")
started_at = time.perf_counter()
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane=lane)
cars_upserted = 0
cars_failed = 0
cars_filtered = 0
images_upserted = 0
total = 0
skipped_existing = 0
failures: list[dict[str, str]] = []
listing: dict = {}
try:
listing = self.collect_listing(make=make, model=model)
vehicle_urls = list(listing.get("vehicle_urls", []))
effective_only_new = self.settings.sync_only_new if only_new is None else only_new
if effective_only_new:
existing_urls = self.persistence.get_existing_origin_urls(vehicle_urls)
url_to_origin_id = {
url: self._extract_origin_id_from_url(url)
for url in vehicle_urls
}
candidate_origin_ids = [origin_id for origin_id in url_to_origin_id.values() if origin_id]
existing_ids = self.persistence.get_existing_origin_ids(candidate_origin_ids)
known_urls = {
url
for url in vehicle_urls
if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids)
}
skipped_existing = len(known_urls)
if skipped_existing:
logger.info("Filtering already known vehicles: skipped %d", skipped_existing)
vehicle_urls = [url for url in vehicle_urls if url not in known_urls]
if limit is not None:
vehicle_urls = vehicle_urls[:max(0, limit)]
total = len(vehicle_urls)
logger.info("Starting sync: %d vehicles to process", total)
for index, vehicle_url in enumerate(vehicle_urls, start=1):
page = self._get_page()
try:
logger.info("[%d/%d] Scraping %s", index, total, vehicle_url)
scrape_result = self._scrape_on_page(page, vehicle_url)
db_record = scrape_result.get("db_record")
if not db_record:
raise RuntimeError("Scrape result does not contain db_record")
record = CarRecord.model_validate(db_record)
# Применяем фильтры из runtime_config (include/exclude/price/mileage/flags)
if not self.runtime_config.filters.is_empty():
vehicle_summary = scrape_result.get("vehicle_summary", {}) or {}
filter_values = {
"brand": record.brand,
"model": record.model,
"year": record.year,
"body_type": record.body_type,
"color": record.color,
"drive": record.drive,
"gearbox": record.gearbox,
"location": vehicle_summary.get("location"),
"price": record.price,
"mileage": record.mileage,
"is_damaged": record.is_damaged,
"run_and_drive": vehicle_summary.get("run_and_drive"),
}
if not self.runtime_config.filters.matches(filter_values):
logger.info(
"[%d/%d] Skipped by filter: %s %s %s",
index, total, record.brand, record.model, record.year or "?",
)
cars_filtered += 1
continue
upsert = self.persistence.upsert_car(record)
if upsert.get("action") != "skipped":
cars_upserted += 1
img_count = int(upsert.get("images_upserted", 0))
images_upserted += img_count
logger.info(
"[%d/%d] %s %s: %s %s %s — %s, %d images",
index, total, upsert.get("action", "?"),
record.origin_id, record.brand, record.model,
record.year or "?", record.price or "N/A", img_count,
)
except Exception as exc:
cars_failed += 1
failures.append({"vehicle_url": vehicle_url, "error": str(exc)})
logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc)
finally:
try:
page.close()
except Exception:
pass
if index < total:
self.pacer.between_vehicles()
except Exception as exc:
if not failures:
failures.append({"vehicle_url": "collect_listing", "error": str(exc)})
logger.error("sync_listing failed: %s", exc)
finally:
status = "success" if not failures else ("partial_success" if cars_upserted else "failed")
error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None
self.persistence.finish_sync_run(
run_id,
status=status,
ids_fetched=total,
cars_upserted=cars_upserted,
cars_failed=cars_failed,
images_upserted=images_upserted,
error_summary=error_summary,
)
logger.info(
"Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images",
run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted,
)
return {
"trace_id": trace_id,
"status": status,
"run_id": run_id,
"listing": listing,
"cars_upserted": cars_upserted,
"cars_failed": cars_failed,
"cars_filtered": cars_filtered,
"images_upserted": images_upserted,
"skipped_existing": skipped_existing,
"elapsed_seconds": round(time.perf_counter() - started_at, 3),
"failures": failures,
}
def run_scheduled(self) -> None:
# NOTE: Зарезервировано для standalone-режима (без Celery beat).
# В текущей архитектуре планирование выполняется через Celery beat + worker/tasks.py.
def _handle_shutdown(signum, frame):
logger.info("Received signal %s, shutting down gracefully...", signum)
self._shutdown_requested = True
signal.signal(signal.SIGINT, _handle_shutdown)
signal.signal(signal.SIGTERM, _handle_shutdown)
interval = self.settings.scheduler_interval_minutes * 60
logger.info(
"Scheduler started: syncing every %d minutes",
self.settings.scheduler_interval_minutes,
)
cycle = 0
while not self._shutdown_requested:
cycle += 1
logger.info("Scheduler cycle #%d starting", cycle)
start = time.time()
try:
if self.context:
try:
self.context.close()
except PlaywrightError:
pass
self.context = None
if self.browser is None or self.playwright is None:
logger.info("Browser/Playwright not available, re-initializing...")
self.close()
self.__enter__()
result = self.sync_listing()
elapsed = time.time() - start
logger.info(
"Cycle #%d done in %.1fs: %d upserted, %d failed",
cycle, elapsed,
result.get("cars_upserted", 0),
result.get("cars_failed", 0),
)
except Exception as exc:
elapsed = time.time() - start
logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc)
try:
self.close()
except Exception:
pass
try:
self.__enter__()
except Exception as reinit_exc:
logger.error("Failed to re-initialize browser: %s", reinit_exc)
if self._shutdown_requested:
break
sleep_time = max(0, interval - (time.time() - start))
if sleep_time > 0:
logger.info("Sleeping %.0f seconds until next cycle...", sleep_time)
slept = 0.0
while slept < sleep_time and not self._shutdown_requested:
chunk = min(5.0, sleep_time - slept)
time.sleep(chunk)
slept += chunk
logger.info("Scheduler stopped gracefully after %d cycles.", cycle)

1
iaai_scraper/storage/__init__.py
View File

@@ -1 +0,0 @@
__all__: list[str] = []

146
iaai_scraper/storage/db.py
View File

@@ -1,146 +0,0 @@
import logging
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Iterator
from sqlalchemy import create_engine, or_, select
from sqlalchemy.orm import Session, sessionmaker
from ..core.config import Settings
from .models import Base, Car, Image, SyncRun
from .schemas import CarRecord
logger = logging.getLogger("iaai_scraper.db")
CAR_DB_FIELDS = {
col.key for col in Car.__table__.columns
if col.key not in ("id",)
}
class PersistenceService:
def __init__(self, settings: Settings) -> None:
self.settings = settings
engine_kwargs = {
"echo": settings.database.echo,
"future": True,
}
if "postgresql" in settings.database.url:
engine_kwargs["pool_size"] = settings.database.pool_size
engine_kwargs["max_overflow"] = settings.database.max_overflow
engine_kwargs["pool_pre_ping"] = True
engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
self.engine = create_engine(settings.database.url, **engine_kwargs)
self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
def create_tables(self) -> None:
# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
is_sqlite = self.settings.database.url.startswith("sqlite")
if not is_sqlite and not self.settings.database.auto_create_tables:
return
try:
Base.metadata.create_all(self.engine)
except Exception:
logger.debug("create_tables skipped (schema already exists)")
@contextmanager
def session_scope(self) -> Iterator[Session]:
session = self.session_factory()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
def start_sync_run(self, lane: str) -> int:
with self.session_scope() as session:
now = datetime.now(timezone.utc)
stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
for stale in stale_runs:
stale.status = "failed"
stale.finished_at = now
if not stale.error_summary:
stale.error_summary = "Recovered stale running sync run before starting a new run"
run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
session.add(run)
session.flush()
return int(run.id)
def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
with self.session_scope() as session:
run = session.get(SyncRun, run_id)
if run is None:
return
run.finished_at = datetime.now(timezone.utc)
run.status = status
run.ids_fetched = ids_fetched
run.cars_upserted = cars_upserted
run.cars_failed = cars_failed
run.images_upserted = images_upserted
run.error_summary = error_summary
def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
if not origin_urls:
return set()
with self.session_scope() as session:
rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all()
return {str(row[0]) for row in rows if row and row[0]}
def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
if not origin_ids:
return set()
with self.session_scope() as session:
rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
return {str(row[0]) for row in rows if row and row[0]}
@staticmethod
def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
for image_payload in images:
session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id))
@staticmethod
def _car_payload(record: CarRecord) -> dict[str, object]:
payload = record.model_dump(mode="python")
return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
def upsert_car(self, record: CarRecord):
# Insert/update автомобиля по origin_id/origin_url.
payload = self._car_payload(record)
images = [image.model_dump(mode="python") for image in record.images]
with self.session_scope() as session:
car = session.execute(
select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
).scalar_one_or_none()
action = "inserted"
if car is None:
car = Car(**payload)
session.add(car)
session.flush()
else:
action = "updated"
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
session.flush()
nested = session.begin_nested()
try:
for image in list(car.images):
session.delete(image)
session.flush()
self._add_images(session, int(car.id), images)
session.flush()
nested.commit()
except Exception:
nested.rollback()
logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id)
images = []
return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
self._add_images(session, int(car.id), images)
session.flush()
return {"car_id": int(car.id), "images_upserted": len(images), "action": action}

205
iaai_scraper/worker/tasks.py
View File

@@ -1,205 +0,0 @@
# Celery-задачи для синхронизации автомобилей и листинга IAAI.
from concurrent.futures import ThreadPoolExecutor
import json
import logging
from celery import shared_task
from redis import Redis
from ..core.config import Settings
from ..scraper import IAAIScraper
from ..storage.db import PersistenceService
logger = logging.getLogger("iaai_scraper.worker.tasks")
SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
def _run_browser_job(func, *args, **kwargs):
# Playwright Sync API нельзя запускать в потоке с активным asyncio loop.
# Celery/зависимости могут поднимать loop в worker-процессе, поэтому
# браузерный код выполняем в отдельном thread без loop.
with ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser") as executor:
future = executor.submit(func, *args, **kwargs)
return future.result()
def _sync_listing_lock_ttl_seconds() -> int:
settings = Settings()
# Небольшой запас к hard time limit задачи, чтобы lock самоснимался после сбоев.
return max(settings.celery.task_time_limit + 120, 300)
def _get_persistence() -> PersistenceService:
return PersistenceService(Settings())
def _get_redis() -> Redis:
settings = Settings()
return Redis.from_url(settings.redis.url, decode_responses=True)
def _release_lock_if_owner(redis_client: Redis, key: str, owner: str) -> None:
try:
current_owner = redis_client.get(key)
if current_owner == owner:
redis_client.delete(key)
except Exception as exc:
logger.warning("Failed to release lock %s: %s", key, exc)
def _has_other_active_sync_listing_task(task) -> bool:
try:
inspector = task.app.control.inspect(timeout=1.0)
active_map = inspector.active() or {}
except Exception as exc:
logger.warning("Failed to inspect active tasks: %s", exc)
return False
current_task_id = task.request.id
for worker_tasks in active_map.values():
for item in worker_tasks or []:
name = str(item.get("name") or "")
task_id = str(item.get("id") or "")
if (
name == "iaai_scraper.worker.tasks.sync_listing_task"
and task_id
and task_id != current_task_id
):
return True
return False
@shared_task(
name="iaai_scraper.worker.tasks.sync_vehicle_task",
bind=True,
max_retries=2,
default_retry_delay=30,
acks_late=True,
)
def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
# Скрапинг и upsert одного автомобиля.
persistence = _get_persistence()
persistence.create_tables()
try:
def _job():
with IAAIScraper() as scraper:
return scraper.sync_vehicle(vehicle_url, lane=lane)
result = _run_browser_job(_job)
logger.info("sync_vehicle_task completed: %s", vehicle_url)
return {
"status": "success",
"vehicle_url": vehicle_url,
"db_action": result.get("db_action"),
"images_upserted": result.get("images_upserted", 0),
}
except Exception as exc:
logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc)
raise self.retry(exc=exc)
@shared_task(
name="iaai_scraper.worker.tasks.sync_listing_task",
bind=True,
max_retries=1,
default_retry_delay=120,
acks_late=True,
)
def sync_listing_task(
self,
make: str | None = None,
model: str | None = None,
lane: str = "iaai_cars",
limit: int | None = None,
only_new: bool | None = None,
):
# Полный цикл: листинг + sync всех найденных машин.
persistence = _get_persistence()
persistence.create_tables()
task_id = self.request.id or "unknown"
redis_client = _get_redis()
lock_acquired = False
lock_ttl = _sync_listing_lock_ttl_seconds()
try:
lock_acquired = bool(
redis_client.set(
SYNC_LISTING_LOCK_KEY,
task_id,
nx=True,
ex=lock_ttl,
)
)
except Exception as exc:
logger.warning("Failed to acquire sync lock in Redis: %s", exc)
if not lock_acquired:
# Возможен stale lock после рестарта worker. Если активного sync_listing нет —
# снимаем lock и пытаемся взять его заново.
if not _has_other_active_sync_listing_task(self):
try:
stale_owner = redis_client.get(SYNC_LISTING_LOCK_KEY)
if stale_owner:
logger.warning(
"Removing stale sync lock held by task %s",
stale_owner,
)
redis_client.delete(SYNC_LISTING_LOCK_KEY)
lock_acquired = bool(
redis_client.set(
SYNC_LISTING_LOCK_KEY,
task_id,
nx=True,
ex=lock_ttl,
)
)
except Exception as exc:
logger.warning("Failed to recover stale sync lock: %s", exc)
if not lock_acquired:
logger.info("sync_listing_task skipped: another sync is already running")
return {
"status": "skipped",
"reason": "sync_already_running",
"task_id": task_id,
}
try:
self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
def _job():
with IAAIScraper() as scraper:
return scraper.sync_listing(
make=make,
model=model,
lane=lane,
limit=limit,
only_new=only_new,
)
result = _run_browser_job(_job)
summary = {
"task_id": task_id,
"run_id": result.get("run_id"),
"cars_upserted": result.get("cars_upserted", 0),
"cars_failed": result.get("cars_failed", 0),
"images_upserted": result.get("images_upserted", 0),
"skipped_existing": result.get("skipped_existing", 0),
"elapsed_seconds": result.get("elapsed_seconds"),
}
logger.info(
"sync_listing_task completed: %d upserted, %d failed",
summary["cars_upserted"], summary["cars_failed"],
)
return {"status": "success", **summary}
except Exception as exc:
logger.error("sync_listing_task failed: %s", exc)
raise self.retry(exc=exc)
finally:
if lock_acquired:
_release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, task_id)

10
pyproject.toml
View File

@@ -3,22 +3,22 @@ requires = ["setuptools>=68", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "iaai-scraper"
name = "encar-scraper"
version = "0.1.0"
description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
description = "Encar scraper service with FastAPI, Celery and PostgreSQL"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
"alembic>=1.14.0",
"celery>=5.4.0",
"fastapi>=0.115.0",
"playwright>=1.53.0",
"psycopg2-binary>=2.9.9",
"pydantic>=2.8.2",
"python-dotenv>=1.0.1",
"redis>=5.2.0",
"sqlalchemy>=2.0.32",
"uvicorn>=0.34.0",
"urllib3>=2.0.0",
]
[project.optional-dependencies]
@@ -28,13 +28,13 @@ dev = [
]
[project.scripts]
iaai = "iaai_scraper.cli:main"
encar = "encar_scraper.cli:main"
[tool.setuptools]
include-package-data = true
[tool.setuptools.packages.find]
include = ["iaai_scraper*"]
include = ["encar_scraper*"]
[tool.pytest.ini_options]
addopts = "-q --disable-warnings"

7
runtime_config.json
View File

@@ -5,9 +5,10 @@
"ids_next_size": null,
"ids_max_pages": null,
"condition_check_enabled": false,
"lane": "iaai_cars",
"only_new": true,
"limit": null
"lane": "encar_cars",
"only_new": false,
"limit": null,
"probe_all_photos": true
},
"filters": {
"price": {

38
tests/test_db.py
View File

@@ -1,4 +1,4 @@
from __future__ import annotations
from __future__ import annotations
import tempfile
import unittest
@@ -6,10 +6,10 @@ from pathlib import Path
from sqlalchemy import select
from iaai_scraper.core.config import Settings
from iaai_scraper.storage.db import PersistenceService
from iaai_scraper.storage.models import Car, Image, SyncRun
from iaai_scraper.storage.schemas import CarRecord, ImageRecord
from encar_scraper.core.config import Settings
from encar_scraper.storage.db import PersistenceService
from encar_scraper.storage.models import Car, Image, SyncRun
from encar_scraper.storage.schemas import CarRecord, ImageRecord
class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
@staticmethod
def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
return CarRecord(
parser_id=f"iaai:{origin_id}",
parser_id=f"encar:{origin_id}",
brand="Toyota",
model="Camry",
year=2014,
price=price,
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
origin_url=f"https://www.encar.com/dc/dc_cardetailview.do?carid={origin_id}",
origin_id=origin_id,
slug=f"toyota-camry-{origin_id}",
images=[
ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
fullres_image=f"https://www.encar.com/carpicture01/pic{origin_id}_001.jpg",
preview_image=f"https://www.encar.com/carpicture01/pic{origin_id}_001.jpg",
order_index=0,
)
],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
second = self._record("888")
second.images = [
ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
fullres_image="https://www.encar.com/carpicture01/pic888_002.jpg",
preview_image="https://www.encar.com/carpicture01/pic888_002.jpg",
order_index=0,
)
]
@@ -90,17 +90,7 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
images = session.execute(select(Image)).scalars().all()
self.assertEqual(len(images), 1)
self.assertIn("imageKeys=2", images[0].fullres_image)
def test_persistence_ignores_non_db_fields(self) -> None:
record = self._record("1000")
result = self.persistence.upsert_car(record)
self.assertEqual(result["action"], "inserted")
with self.persistence.session_scope() as session:
car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one()
self.assertEqual(car.origin_id, "1000")
self.assertIn("pic888_002.jpg", images[0].fullres_image)
def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
first_run_id = self.persistence.start_sync_run("lane-a")
@@ -118,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
first = self._record("OLD-ID")
first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
first.origin_url = "https://www.encar.com/dc/dc_cardetailview.do?carid=45089484"
self.persistence.upsert_car(first)
second = self._record("NEW-ID")
second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
second.origin_url = "https://www.encar.com/dc/dc_cardetailview.do?carid=45089484"
result = self.persistence.upsert_car(second)
self.assertEqual(result["action"], "updated")

62
tests/test_encar.py Normal file
View File

@@ -0,0 +1,62 @@
import unittest
from unittest.mock import MagicMock
from encar_scraper.encar import EncarMapper, EncarScraper, ENCAR_DETAIL_URL_TEMPLATE
class TestEncarMapper(unittest.TestCase):
def test_map_to_car_record(self):
payload = {
"Id": 41421262,
"Manufacturer": "Hyundai",
"Model": "G90 (RS4)",
"FormYear": "2023",
"Price": 8590,
"Mileage": 37923,
"Photos": [
{"location": "/carpicture01/pic4141/41412663_001.jpg", "ordering": 1},
{"location": "/carpicture01/pic4141/41412663_003.jpg", "ordering": 3},
],
}
record = EncarMapper().map_to_car_record(
ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41421262"),
payload,
)
self.assertEqual(record.origin_id, "encar:41421262")
self.assertEqual(record.parser_id, "encar:41421262")
self.assertEqual(record.brand, "Hyundai")
self.assertEqual(record.model, "G90 (RS4)")
self.assertEqual(record.year, 2023)
self.assertEqual(record.price, 85900000) # 8590 만원 × 10000
self.assertEqual(record.mileage, 37923)
self.assertEqual(record.country, "KR")
self.assertEqual(record.currency, "KRW")
self.assertEqual(len(record.images), 2)
self.assertTrue(record.images[0].fullres_image.endswith("_001.jpg"))
def test_extract_vehicle_id_from_url(self):
url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="41421262")
self.assertEqual(EncarMapper()._extract_vehicle_id(url), "41421262")
class TestEncarScraper(unittest.TestCase):
def test_collect_listing_limits_results(self):
scraper = EncarScraper()
scraper._fetch_listing_page = MagicMock(side_effect=[
{"SearchResults": [{"Id": 1}, {"Id": 2}]},
])
result = scraper.collect_listing(limit=1, page_size=2)
self.assertEqual(result["items_collected"], 1)
self.assertEqual(result["vehicle_urls"], [ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id="1")])
def test_scrape_vehicle_detail_raises_without_id(self):
scraper = EncarScraper()
with self.assertRaises(ValueError):
scraper.scrape_vehicle_detail("https://www.encar.com/invalid")
if __name__ == "__main__":
unittest.main()

42
tests/test_listing.py
View File

@@ -1,42 +0,0 @@
from __future__ import annotations
import unittest
from iaai_scraper.browser.pace import HumanPacer
from iaai_scraper.core.config import Settings
from iaai_scraper.browser.listing import ListingCollector
class _FakePage:
def __init__(self, counts: dict[str, int]) -> None:
self._counts = counts
class _Locator:
def __init__(self, count_value: int) -> None:
self._count_value = count_value
def count(self) -> int:
return self._count_value
def locator(self, selector: str) -> "_FakePage._Locator":
return _FakePage._Locator(self._counts.get(selector, 0))
class TestListingUnit(unittest.TestCase):
def test_has_next_page_true_for_known_selector(self) -> None:
page = _FakePage({"a[aria-label*='Next']": 1})
self.assertTrue(ListingCollector._has_next_page(page))
def test_has_next_page_false_when_no_selectors(self) -> None:
page = _FakePage({})
self.assertFalse(ListingCollector._has_next_page(page))
def test_constructor_with_settings_and_pacer(self) -> None:
settings = Settings()
pacer = HumanPacer(settings)
collector = ListingCollector(settings, pacer)
self.assertIsNotNone(collector)
if __name__ == "__main__":
unittest.main()

414
tests/test_load.py Normal file
View File

@@ -0,0 +1,414 @@
"""
Нагрузочные тесты: API endpoints, batch upsert, конкурентный доступ к БД.
Запуск:
pytest tests/test_load.py -v -s
"""
from __future__ import annotations
import concurrent.futures
import time
import random
import string
import unittest
from datetime import datetime, timezone
from fastapi.testclient import TestClient
from encar_scraper.api.app import create_app
from encar_scraper.core.config import DatabaseConfig, RedisConfig, Settings
from encar_scraper.storage.db import PersistenceService
from encar_scraper.storage.schemas import CarRecord, ImageRecord
# ── helpers ──────────────────────────────────────────────────────────────
def _sqlite_settings() -> Settings:
return Settings(
database=DatabaseConfig(url="sqlite:///test_load.db", echo=False, auto_create_tables=True),
redis=RedisConfig(url="redis://localhost:6379/15"),
)
def _make_car(idx: int, *, images: int = 3) -> CarRecord:
uid = f"load-{idx}-{''.join(random.choices(string.ascii_lowercase, k=4))}"
return CarRecord(
parser_id=uid,
brand=random.choice(["Hyundai", "Kia", "BMW", "Mercedes", "Genesis"]),
model=f"Model-{idx % 50}",
year=random.randint(2010, 2025),
price=random.randint(5_000_000, 80_000_000),
currency="KRW",
mileage=random.randint(0, 300_000),
country="KR",
origin="ENCAR",
origin_url=f"https://encar.com/cars/{uid}",
origin_id=uid,
slug=uid,
selling_type="NA",
body_type="SEDAN",
last_seen_at=datetime.now(timezone.utc),
images=[
ImageRecord(
fullres_image=f"https://ci.encar.com/photo/{uid}/{j}.jpg",
preview_image=f"https://ci.encar.com/photo/{uid}/{j}_thumb.jpg",
order_index=j,
)
for j in range(images)
],
)
# ── DB load tests ───────────────────────────────────────────────────────
class TestDatabaseLoad(unittest.TestCase):
"""Тесты производительности PersistenceService на SQLite."""
@classmethod
def setUpClass(cls):
import os
if os.path.exists("test_load.db"):
os.remove("test_load.db")
cls.settings = _sqlite_settings()
cls.persistence = PersistenceService(cls.settings)
cls.persistence.create_tables()
@classmethod
def tearDownClass(cls):
cls.persistence.engine.dispose()
import os
if os.path.exists("test_load.db"):
os.remove("test_load.db")
def test_batch_upsert_500_cars(self):
"""Пакетная вставка 500 авто с изображениями — должна пройти за <10 сек."""
records = [_make_car(i, images=5) for i in range(500)]
t0 = time.perf_counter()
result = self.persistence.upsert_cars_batch(records)
elapsed = time.perf_counter() - t0
print(f"\n[batch_upsert_500] inserted={result['inserted']}, "
f"updated={result['updated']}, images={result['images_upserted']}, "
f"time={elapsed:.2f}s")
self.assertEqual(result["inserted"], 500)
self.assertEqual(result["images_upserted"], 2500)
self.assertLess(elapsed, 10, "Batch upsert 500 cars should complete in <10s")
def test_batch_upsert_update_existing(self):
"""Повторный upsert 200 существующих записей — обновление без дублей."""
records = [_make_car(i, images=3) for i in range(10000, 10200)]
self.persistence.upsert_cars_batch(records)
# Обновляем цену и перезаписываем
for r in records:
r.price = 99_999_999
t0 = time.perf_counter()
result = self.persistence.upsert_cars_batch(records)
elapsed = time.perf_counter() - t0
print(f"\n[batch_update_200] updated={result['updated']}, time={elapsed:.2f}s")
self.assertEqual(result["updated"], 200)
self.assertLess(elapsed, 10)
def test_concurrent_upsert_batches(self):
"""3 параллельных потока по 100 upsert-ов — нет deadlock / corruption."""
def worker(thread_id: int):
batch = [_make_car(thread_id * 10000 + i) for i in range(100)]
return self.persistence.upsert_cars_batch(batch)
t0 = time.perf_counter()
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as pool:
futures = [pool.submit(worker, tid) for tid in range(3)]
results = [f.result() for f in concurrent.futures.as_completed(futures)]
elapsed = time.perf_counter() - t0
total_inserted = sum(r["inserted"] for r in results)
print(f"\n[concurrent_upsert] total_inserted={total_inserted}, time={elapsed:.2f}s")
self.assertEqual(total_inserted, 300)
self.assertLess(elapsed, 30)
def test_single_upsert_throughput(self):
"""50 последовательных upsert_car — замер throughput."""
records = [_make_car(50000 + i, images=2) for i in range(50)]
t0 = time.perf_counter()
for r in records:
self.persistence.upsert_car(r)
elapsed = time.perf_counter() - t0
rps = 50 / elapsed if elapsed > 0 else 0
print(f"\n[single_upsert_50] time={elapsed:.2f}s, rps={rps:.1f}")
self.assertLess(elapsed, 15, "50 individual upserts should complete in <15s")
def test_mark_sold_large_set(self):
"""mark_sold на 300 origin_id — должен быть быстрым."""
records = [_make_car(60000 + i, images=1) for i in range(300)]
self.persistence.upsert_cars_batch(records)
active_ids = {r.origin_id for r in records[:150]}
t0 = time.perf_counter()
sold = self.persistence.mark_sold_not_in_listing(active_origin_ids=active_ids, lane="encar")
elapsed = time.perf_counter() - t0
print(f"\n[mark_sold_300] marked_sold={sold}, time={elapsed:.2f}s")
self.assertLess(elapsed, 5)
def test_sync_run_lifecycle(self):
"""start → finish sync_run 50 раз — измеряем overhead."""
t0 = time.perf_counter()
for _ in range(50):
run_id = self.persistence.start_sync_run("encar")
self.persistence.finish_sync_run(
run_id,
status="completed",
ids_fetched=1000,
cars_upserted=950,
cars_failed=50,
images_upserted=15000,
)
elapsed = time.perf_counter() - t0
print(f"\n[sync_run_lifecycle_50] time={elapsed:.2f}s")
self.assertLess(elapsed, 5)
# ── API load tests ──────────────────────────────────────────────────────
class TestAPILoad(unittest.TestCase):
"""Нагрузочные тесты на FastAPI endpoints через TestClient."""
@classmethod
def setUpClass(cls):
import os
if os.path.exists("test_api_load.db"):
os.remove("test_api_load.db")
settings = Settings(
database=DatabaseConfig(url="sqlite:///test_api_load.db", echo=False, auto_create_tables=True),
redis=RedisConfig(url="redis://localhost:6379/15"),
)
app = create_app(settings)
app.state.persistence.create_tables()
# Предзаполняем БД 200 авто
persistence: PersistenceService = app.state.persistence
persistence.upsert_cars_batch([_make_car(i, images=4) for i in range(200)])
cls.client = TestClient(app)
@classmethod
def tearDownClass(cls):
import os
if os.path.exists("test_api_load.db"):
os.remove("test_api_load.db")
def test_health_check_rapid(self):
"""100 GET /health подряд — все 200, <5 сек."""
t0 = time.perf_counter()
for _ in range(100):
r = self.client.get("/health")
self.assertEqual(r.status_code, 200)
elapsed = time.perf_counter() - t0
print(f"\n[health_100] time={elapsed:.2f}s, rps={100 / elapsed:.0f}")
self.assertLess(elapsed, 5)
def test_list_cars_paginated_rapid(self):
"""100 GET /api/v1/cars с разными страницами — все 200, <10 сек."""
t0 = time.perf_counter()
for page in range(1, 101):
r = self.client.get("/api/v1/cars", params={"page": (page % 10) + 1, "per_page": 20})
self.assertEqual(r.status_code, 200)
data = r.json()
self.assertIn("items", data)
elapsed = time.perf_counter() - t0
print(f"\n[list_cars_100] time={elapsed:.2f}s, rps={100 / elapsed:.0f}")
self.assertLess(elapsed, 10)
def test_list_cars_with_filters(self):
"""50 GET /api/v1/cars с brand/year фильтрами."""
t0 = time.perf_counter()
for i in range(50):
brand = random.choice(["Hyundai", "Kia", "BMW", "Mercedes", "Genesis"])
r = self.client.get(
"/api/v1/cars",
params={"brand": brand, "year_min": 2015, "year_max": 2023, "per_page": 50},
)
self.assertEqual(r.status_code, 200)
elapsed = time.perf_counter() - t0
print(f"\n[filtered_cars_50] time={elapsed:.2f}s, rps={50 / elapsed:.0f}")
self.assertLess(elapsed, 10)
def test_get_car_by_id_rapid(self):
"""50 GET /api/v1/cars/{id} — все 200."""
t0 = time.perf_counter()
for car_id in range(1, 51):
r = self.client.get(f"/api/v1/cars/{car_id}")
self.assertIn(r.status_code, (200, 404))
elapsed = time.perf_counter() - t0
print(f"\n[get_car_50] time={elapsed:.2f}s, rps={50 / elapsed:.0f}")
self.assertLess(elapsed, 10)
def test_stats_endpoint_rapid(self):
"""50 GET /api/v1/stats — агрегация по всей таблице."""
t0 = time.perf_counter()
for _ in range(50):
r = self.client.get("/api/v1/stats")
self.assertEqual(r.status_code, 200)
data = r.json()
self.assertGreater(data["total_cars"], 0)
elapsed = time.perf_counter() - t0
print(f"\n[stats_50] time={elapsed:.2f}s, rps={50 / elapsed:.0f}")
self.assertLess(elapsed, 10)
def test_concurrent_api_requests(self):
"""20 конкурентных запросов к разным endpoints — нет 500-х."""
def make_request(idx: int):
if idx % 3 == 0:
return self.client.get("/health")
elif idx % 3 == 1:
return self.client.get("/api/v1/cars", params={"page": 1, "per_page": 10})
else:
return self.client.get("/api/v1/stats")
t0 = time.perf_counter()
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool:
futures = [pool.submit(make_request, i) for i in range(20)]
responses = [f.result() for f in concurrent.futures.as_completed(futures)]
elapsed = time.perf_counter() - t0
for r in responses:
self.assertNotEqual(r.status_code, 500, f"Got 500: {r.text}")
print(f"\n[concurrent_api_20] time={elapsed:.2f}s, all_ok={all(r.status_code == 200 for r in responses)}")
self.assertLess(elapsed, 10)
def test_large_page_size(self):
"""GET /api/v1/cars с per_page=100 — проверяем сериализацию больших ответов."""
t0 = time.perf_counter()
for _ in range(20):
r = self.client.get("/api/v1/cars", params={"page": 1, "per_page": 100})
self.assertEqual(r.status_code, 200)
data = r.json()
# Каждый car содержит images — проверяем что сериализация прошла
for item in data["items"]:
self.assertIn("images", item)
elapsed = time.perf_counter() - t0
print(f"\n[large_page_20] time={elapsed:.2f}s, rps={20 / elapsed:.0f}")
self.assertLess(elapsed, 10)
# ── Data integrity under load ───────────────────────────────────────────
class TestDataIntegrity(unittest.TestCase):
"""Проверка целостности данных при массовых операциях."""
@classmethod
def setUpClass(cls):
import os
if os.path.exists("test_integrity.db"):
os.remove("test_integrity.db")
cls.settings = Settings(
database=DatabaseConfig(url="sqlite:///test_integrity.db", echo=False, auto_create_tables=True),
redis=RedisConfig(url="redis://localhost:6379/15"),
)
cls.persistence = PersistenceService(cls.settings)
cls.persistence.create_tables()
@classmethod
def tearDownClass(cls):
cls.persistence.engine.dispose()
import os
if os.path.exists("test_integrity.db"):
os.remove("test_integrity.db")
def test_no_duplicates_after_repeated_upserts(self):
"""3 раза upsert одного батча — ровно N уникальных записей в БД."""
records = [_make_car(90000 + i) for i in range(100)]
for _ in range(3):
self.persistence.upsert_cars_batch(records)
from sqlalchemy import select, func
from encar_scraper.storage.models import Car
with self.persistence.session_scope() as session:
count = session.execute(
select(func.count(Car.id)).where(Car.origin_id.like("load-90%"))
).scalar()
self.assertEqual(count, 100, f"Expected 100 unique cars, got {count}")
def test_images_replaced_not_accumulated(self):
"""При upsert с новыми images — старые удаляются, а не копятся."""
rec = _make_car(80001, images=5)
self.persistence.upsert_car(rec)
# Обновляем с 2 изображениями
rec.images = [
ImageRecord(fullres_image="https://new/1.jpg", preview_image="https://new/1t.jpg", order_index=0),
ImageRecord(fullres_image="https://new/2.jpg", preview_image="https://new/2t.jpg", order_index=1),
]
self.persistence.upsert_car(rec)
from sqlalchemy import select, func
from encar_scraper.storage.models import Car, Image
with self.persistence.session_scope() as session:
car = session.execute(select(Car).where(Car.origin_id == rec.origin_id)).scalar_one()
img_count = session.execute(
select(func.count(Image.id)).where(Image.car_id == car.id)
).scalar()
self.assertEqual(img_count, 2, f"Expected 2 images after replace, got {img_count}")
def test_dedup_within_batch(self):
"""Батч с дубликатами origin_id — дедупликация работает."""
base = _make_car(70001)
dupe = _make_car(70001)
dupe.parser_id = base.parser_id
dupe.origin_id = base.origin_id
dupe.origin_url = base.origin_url
dupe.price = 12345
records = [base, dupe]
result = self.persistence.upsert_cars_batch(records)
self.assertEqual(result["inserted"] + result["updated"], 1,
"Dedup should reduce 2 records to 1")
def test_batch_1000_cars_integrity(self):
"""Вставка 1000 авто — проверяем что ВСЕ записи в БД с верными images."""
records = [_make_car(200000 + i, images=3) for i in range(1000)]
result = self.persistence.upsert_cars_batch(records)
self.assertEqual(result["inserted"], 1000)
self.assertEqual(result["images_upserted"], 3000)
from sqlalchemy import select, func
from encar_scraper.storage.models import Car, Image
with self.persistence.session_scope() as session:
car_count = session.execute(
select(func.count(Car.id)).where(Car.origin_id.like("load-20____-%"))
).scalar()
img_count = session.execute(
select(func.count(Image.id)).where(
Image.car_id.in_(
select(Car.id).where(Car.origin_id.like("load-20____-%"))
)
)
).scalar()
self.assertEqual(car_count, 1000)
self.assertEqual(img_count, 3000)
if __name__ == "__main__":
unittest.main()

100
tests/test_mappers.py
View File

@@ -1,100 +0,0 @@
from __future__ import annotations
import unittest
from iaai_scraper.parsing.mapper import CarMapper
class TestCarMapper(unittest.TestCase):
def setUp(self) -> None:
self.mapper = CarMapper()
def test_deduplicates_images_by_image_key(self) -> None:
urls = [
"https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
"https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
]
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(len(record.images), 2)
self.assertIn("width=845", record.images[0].fullres_image)
self.assertIn("height=633", record.images[0].fullres_image)
def test_no_damage_marker_is_not_damaged(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
vehicle_summary={"make": "Ford", "model": "Focus"},
payload_insights={
"vehicle_core": {},
"pricing": {},
"damage": {"primary": "normal wear"},
"auction": {},
"images": {},
},
)
self.assertFalse(record.is_damaged)
def test_unknown_and_empty_values_fallbacks(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.brand, "UNKNOWN")
self.assertEqual(record.model, "UNKNOWN")
self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
self.assertEqual(record.drive, "NA")
self.assertEqual(record.gearbox, "NA")
def test_mapper_handles_case_and_spaces(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.drive, "FWD")
self.assertEqual(record.gearbox, "AT")
def test_price_parsing_dirty_formats(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={
"vehicle_core": {},
"pricing": {"buy_now": "USD 4,500 - 5,200"},
"damage": {},
"auction": {},
"images": {},
},
)
self.assertEqual(record.price, 5200)
def test_currency_detection_from_symbol(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={
"vehicle_core": {},
"pricing": {"buy_now": "€4.500,00"},
"damage": {},
"auction": {},
"images": {},
},
)
self.assertEqual(record.currency, "EUR")
self.assertEqual(record.price, 4500)
if __name__ == "__main__":
unittest.main()

68
tests/test_parser.py
View File

@@ -1,68 +0,0 @@
from __future__ import annotations
import unittest
from iaai_scraper.parsing.parser import VehicleParser
class TestVehicleParserUnit(unittest.TestCase):
def setUp(self) -> None:
self.parser = VehicleParser()
def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
dom_text = """
Stock #:
45089484
VIN (Status):
1HGCM82633A123456 (OK)
Primary Damage:
Front End
"""
result = self.parser._parse_dom_key_value_pairs(dom_text)
self.assertEqual(result.get("lot_number"), "45089484")
self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)")
self.assertEqual(result.get("primary_damage"), "Front End")
def test_parse_title_for_year_make_model(self) -> None:
parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
self.assertEqual(parsed["year"], "2014")
self.assertEqual(parsed["make"], "TOYOTA")
self.assertEqual(parsed["model"], "CAMRY")
def test_extract_image_urls_filters_other_vehicle(self) -> None:
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
payloads = [
{
"imageUrls": [
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
]
}
]
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
self.assertEqual(len(urls), 1)
self.assertIn("45089484", urls[0])
def test_extract_image_urls_deduplicates_same_url(self) -> None:
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
payloads = [{"imageUrls": [
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
]}]
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
self.assertEqual(len(urls), 1)
def test_dom_hints_detect_captcha_and_antibot(self) -> None:
hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
self.assertTrue(hints["has_captcha_text"])
self.assertTrue(hints["has_antibot_text"])
def test_access_notes_reflect_antibot_signals(self) -> None:
summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
notes = self.parser._build_access_notes(summary, [])
self.assertTrue(notes["possible_captcha"])
self.assertTrue(notes["possible_antibot"])
if __name__ == "__main__":
unittest.main()

177
tests/test_scraper.py
View File

@@ -1,177 +0,0 @@
from __future__ import annotations
import unittest
from unittest.mock import MagicMock
from iaai_scraper.core.config import Settings
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.storage.schemas import CarRecord
def make_db_record(origin_id: str) -> dict[str, object]:
return CarRecord(
parser_id=f"iaai:{origin_id}",
brand="Toyota",
model="Camry",
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id,
slug=f"toyota-camry-{origin_id}",
).model_dump(mode="json")
class TestScraperSync(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper:
s = Settings()
s.log_level = "CRITICAL"
s.database.url = "sqlite://"
return IAAIScraper(s)
def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=1)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
self.assertEqual(result["status"], "success")
self.assertIn("trace_id", result)
self.assertIn("elapsed_seconds", result)
self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
def test_sync_listing_uses_db_record_without_remapping(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=2)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
page = MagicMock()
scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
scraper._get_page = MagicMock(return_value=page)
scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
result = scraper.sync_listing()
self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(result["cars_failed"], 0)
self.assertIn("trace_id", result)
self.assertIn("elapsed_seconds", result)
self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
page.close.assert_called_once()
def test_sync_listing_respects_limit(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=3)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
]
})
scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
scraper._get_page = MagicMock(return_value=MagicMock())
scraper.sync_listing(limit=1)
self.assertEqual(scraper._scrape_on_page.call_count, 1)
def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=4)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
scraper._get_page = MagicMock(return_value=MagicMock())
result = scraper.sync_listing()
self.assertEqual(result["cars_upserted"], 0)
def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=5)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"})
scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"})
scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [
"https://www.iaai.com/VehicleDetail/111~US", # exists by URL
"https://www.iaai.com/VehicleDetail/222~US", # exists by ID
"https://www.iaai.com/VehicleDetail/333~US", # new
]
})
page = MagicMock()
scraper._get_page = MagicMock(return_value=page)
scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")})
result = scraper.sync_listing(only_new=True)
self.assertEqual(result["skipped_existing"], 2)
self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(scraper._scrape_on_page.call_count, 1)
scraper.persistence.get_existing_origin_urls.assert_called_once()
scraper.persistence.get_existing_origin_ids.assert_called_once()
def test_close_resets_browser_state(self) -> None:
scraper = self._make_scraper()
scraper.context = MagicMock()
scraper.browser = MagicMock()
scraper.playwright = MagicMock()
scraper.close()
self.assertIsNone(scraper.context)
self.assertIsNone(scraper.browser)
self.assertIsNone(scraper.playwright)
def test_guard_raises_on_antibot_signals(self) -> None:
with self.assertRaises(AntiBotDetectedError):
IAAIScraper._raise_if_blocked_or_incomplete(
{
"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
"access_notes": {},
"vehicle_summary": {},
},
"https://www.iaai.com/VehicleDetail/999~US",
)
def test_guard_raises_on_empty_vehicle_page(self) -> None:
with self.assertRaises(SiteStructureChangedError):
IAAIScraper._raise_if_blocked_or_incomplete(
{
"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
"access_notes": {"possible_captcha": False, "possible_antibot": False},
"vehicle_summary": {},
},
"https://www.iaai.com/VehicleDetail/999~US",
)
if __name__ == "__main__":
unittest.main()

4
tests/test_utils.py
View File

@@ -1,8 +1,8 @@
from __future__ import annotations
from __future__ import annotations
import unittest
from iaai_scraper.core.utils import deep_find_key
from encar_scraper.core.utils import deep_find_key
class TestDeepFindKey(unittest.TestCase):

95
tests/test_worker_tasks.py Normal file
View File

@@ -0,0 +1,95 @@
from __future__ import annotations
import unittest
from unittest.mock import MagicMock, patch
from encar_scraper.worker import tasks
class TestWorkerTaskLockHelpers(unittest.TestCase):
def test_acquire_lock_returns_true_on_success(self) -> None:
redis_client = MagicMock()
redis_client.set.return_value = True
acquired = tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)
self.assertTrue(acquired)
redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
def test_refresh_lock_if_owner_extends_ttl(self) -> None:
redis_client = MagicMock()
mock_script = MagicMock(return_value=1)
redis_client.register_script.return_value = mock_script
refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)
self.assertTrue(refreshed)
redis_client.register_script.assert_called_once()
mock_script.assert_called_once_with(keys=["lock:key"], args=["owner-token", 120])
def test_release_lock_if_owner_uses_owner_token(self) -> None:
redis_client = MagicMock()
mock_script = MagicMock(return_value=1)
redis_client.register_script.return_value = mock_script
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
redis_client.register_script.assert_called_once()
mock_script.assert_called_once_with(keys=["lock:key"], args=["owner-token"])
def test_encar_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=False):
persistence = MagicMock()
get_persistence.return_value = persistence
get_redis.return_value = MagicMock()
tasks.encar_sync_listing_task.push_request(id="task-123")
try:
result = tasks.encar_sync_listing_task.run()
finally:
tasks.encar_sync_listing_task.pop_request()
persistence.create_tables.assert_called_once()
self.assertEqual(result["status"], "skipped")
self.assertEqual(result["reason"], "sync_already_running")
def test_encar_sync_listing_task_releases_owned_lock(self) -> None:
mock_scraper_instance = MagicMock()
mock_scraper_instance.sync_listing.return_value = {
"total_available": 100,
"items_collected": 10,
"cars_synced": 8,
"cars_failed": 2,
}
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch("encar_scraper.worker.tasks.EncarScraper", return_value=mock_scraper_instance):
persistence = MagicMock()
get_persistence.return_value = persistence
redis_client = MagicMock()
get_redis.return_value = redis_client
stop_event = MagicMock()
heartbeat_thread = MagicMock()
start_heartbeat.return_value = (stop_event, heartbeat_thread)
tasks.encar_sync_listing_task.push_request(id="task-123")
try:
with patch.object(tasks.encar_sync_listing_task, "update_state"):
result = tasks.encar_sync_listing_task.run(car_type="all")
finally:
tasks.encar_sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
stop_event.set.assert_called_once()
heartbeat_thread.join.assert_called_once()
release_lock.assert_called_once()
if __name__ == "__main__":
unittest.main()

130
uv.lock generated
View File

@@ -259,6 +259,46 @@ toml = [
{ name = "tomli", marker = "python_full_version <= '3.11'" },
]
[[package]]
name = "encar-scraper"
version = "0.1.0"
source = { editable = "." }
dependencies = [
{ name = "alembic" },
{ name = "celery" },
{ name = "fastapi" },
{ name = "psycopg2-binary" },
{ name = "pydantic" },
{ name = "python-dotenv" },
{ name = "redis" },
{ name = "sqlalchemy" },
{ name = "urllib3" },
{ name = "uvicorn" },
]
[package.optional-dependencies]
dev = [
{ name = "pytest" },
{ name = "pytest-cov" },
]
[package.metadata]
requires-dist = [
{ name = "alembic", specifier = ">=1.14.0" },
{ name = "celery", specifier = ">=5.4.0" },
{ name = "fastapi", specifier = ">=0.115.0" },
{ name = "psycopg2-binary", specifier = ">=2.9.9" },
{ name = "pydantic", specifier = ">=2.8.2" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
{ name = "python-dotenv", specifier = ">=1.0.1" },
{ name = "redis", specifier = ">=5.2.0" },
{ name = "sqlalchemy", specifier = ">=2.0.32" },
{ name = "urllib3", specifier = ">=2.0.0" },
{ name = "uvicorn", specifier = ">=0.34.0" },
]
provides-extras = ["dev"]
[[package]]
name = "fastapi"
version = "0.135.3"
@@ -284,9 +324,7 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/fb/c6/dba32cab7e3a625b011aa5647486e2d28423a48845a2998c126dd69c85e1/greenlet-3.4.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:805bebb4945094acbab757d34d6e1098be6de8966009ab9ca54f06ff492def58", size = 285504, upload-time = "2026-04-08T15:52:14.071Z" },
{ url = "https://files.pythonhosted.org/packages/54/f4/7cb5c2b1feb9a1f50e038be79980dfa969aa91979e5e3a18fdbcfad2c517/greenlet-3.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:439fc2f12b9b512d9dfa681c5afe5f6b3232c708d13e6f02c845e0d9f4c2d8c6", size = 605476, upload-time = "2026-04-08T16:24:37.064Z" },
{ url = "https://files.pythonhosted.org/packages/d6/af/b66ab0b2f9a4c5a867c136bf66d9599f34f21a1bcca26a2884a29c450bd9/greenlet-3.4.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70ed1cb0295bee1df57b63bf7f46b4e56a5c93709eea769c1fec1bb23a95875", size = 618336, upload-time = "2026-04-08T16:30:56.59Z" },
{ url = "https://files.pythonhosted.org/packages/6d/31/56c43d2b5de476f77d36ceeec436328533bff960a4cba9a07616e93063ab/greenlet-3.4.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c5696c42e6bb5cfb7c6ff4453789081c66b9b91f061e5e9367fa15792644e76", size = 625045, upload-time = "2026-04-08T16:40:37.111Z" },
{ url = "https://files.pythonhosted.org/packages/e5/5c/8c5633ece6ba611d64bf2770219a98dd439921d6424e4e8cf16b0ac74ea5/greenlet-3.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c660bce1940a1acae5f51f0a064f1bc785d07ea16efcb4bc708090afc4d69e83", size = 613515, upload-time = "2026-04-08T15:56:32.478Z" },
{ url = "https://files.pythonhosted.org/packages/80/ca/704d4e2c90acb8bdf7ae593f5cbc95f58e82de95cc540fb75631c1054533/greenlet-3.4.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:89995ce5ddcd2896d89615116dd39b9703bfa0c07b583b85b89bf1b5d6eddf81", size = 419745, upload-time = "2026-04-08T16:43:04.022Z" },
{ url = "https://files.pythonhosted.org/packages/a9/df/950d15bca0d90a0e7395eb777903060504cdb509b7b705631e8fb69ff415/greenlet-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee407d4d1ca9dc632265aee1c8732c4a2d60adff848057cdebfe5fe94eb2c8a2", size = 1574623, upload-time = "2026-04-08T16:26:18.596Z" },
{ url = "https://files.pythonhosted.org/packages/1a/e7/0839afab829fcb7333c9ff6d80c040949510055d2d4d63251f0d1c7c804e/greenlet-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:956215d5e355fffa7c021d168728321fd4d31fd730ac609b1653b450f6a4bc71", size = 1639579, upload-time = "2026-04-08T15:57:29.231Z" },
{ url = "https://files.pythonhosted.org/packages/d9/2b/b4482401e9bcaf9f5c97f67ead38db89c19520ff6d0d6699979c6efcc200/greenlet-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cb614ace7c27571270354e9c9f696554d073f8aa9319079dcba466bbdead711", size = 238233, upload-time = "2026-04-08T17:02:54.286Z" },
@@ -294,9 +332,7 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/65/8b/3669ad3b3f247a791b2b4aceb3aa5a31f5f6817bf547e4e1ff712338145a/greenlet-3.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1a54a921561dd9518d31d2d3db4d7f80e589083063ab4d3e2e950756ef809e1a", size = 286902, upload-time = "2026-04-08T15:52:12.138Z" },
{ url = "https://files.pythonhosted.org/packages/38/3e/3c0e19b82900873e2d8469b590a6c4b3dfd2b316d0591f1c26b38a4879a5/greenlet-3.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16dec271460a9a2b154e3b1c2fa1050ce6280878430320e85e08c166772e3f97", size = 606099, upload-time = "2026-04-08T16:24:38.408Z" },
{ url = "https://files.pythonhosted.org/packages/b5/33/99fef65e7754fc76a4ed14794074c38c9ed3394a5bd129d7f61b705f3168/greenlet-3.4.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90036ce224ed6fe75508c1907a77e4540176dcf0744473627785dd519c6f9996", size = 618837, upload-time = "2026-04-08T16:30:58.298Z" },
{ url = "https://files.pythonhosted.org/packages/44/57/eae2cac10421feae6c0987e3dc106c6d86262b1cb379e171b017aba893a6/greenlet-3.4.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6f0def07ec9a71d72315cf26c061aceee53b306c36ed38c35caba952ea1b319d", size = 624901, upload-time = "2026-04-08T16:40:38.981Z" },
{ url = "https://files.pythonhosted.org/packages/36/f7/229f3aed6948faa20e0616a0b8568da22e365ede6a54d7d369058b128afd/greenlet-3.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c4f6b453006efb8310affb2d132832e9bbb4fc01ce6df6b70d810d38f1f6dc", size = 615062, upload-time = "2026-04-08T15:56:33.766Z" },
{ url = "https://files.pythonhosted.org/packages/6a/8a/0e73c9b94f31d1cc257fe79a0eff621674141cdae7d6d00f40de378a1e42/greenlet-3.4.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:0e1254cf0cbaa17b04320c3a78575f29f3c161ef38f59c977108f19ffddaf077", size = 423927, upload-time = "2026-04-08T16:43:05.293Z" },
{ url = "https://files.pythonhosted.org/packages/08/97/d988180011aa40135c46cd0d0cf01dd97f7162bae14139b4a3ef54889ba5/greenlet-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b2d9a138ffa0e306d0e2b72976d2fb10b97e690d40ab36a472acaab0838e2de", size = 1573511, upload-time = "2026-04-08T16:26:20.058Z" },
{ url = "https://files.pythonhosted.org/packages/d4/0f/a5a26fe152fb3d12e6a474181f6e9848283504d0afd095f353d85726374b/greenlet-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8424683caf46eb0eb6f626cb95e008e8cc30d0cb675bdfa48200925c79b38a08", size = 1640396, upload-time = "2026-04-08T15:57:30.88Z" },
{ url = "https://files.pythonhosted.org/packages/42/cf/bb2c32d9a100e36ee9f6e38fad6b1e082b8184010cb06259b49e1266ca01/greenlet-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0a53fb071531d003b075c444014ff8f8b1a9898d36bb88abd9ac7b3524648a2", size = 238892, upload-time = "2026-04-08T17:03:10.094Z" },
@@ -304,9 +340,7 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" },
{ url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" },
{ url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" },
{ url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" },
{ url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" },
{ url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" },
{ url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" },
{ url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" },
{ url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" },
@@ -314,9 +348,7 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" },
{ url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" },
{ url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" },
{ url = "https://files.pythonhosted.org/packages/d2/4a/74078d3936712cff6d3c91a930016f476ce4198d84e224fe6d81d3e02880/greenlet-3.4.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:06c2d3b89e0c62ba50bd7adf491b14f39da9e7e701647cb7b9ff4c99bee04b19", size = 673405, upload-time = "2026-04-08T16:40:42.527Z" },
{ url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" },
{ url = "https://files.pythonhosted.org/packages/79/3e/df8a83ab894751bc31e1106fdfaa80ca9753222f106b04de93faaa55feb7/greenlet-3.4.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:070b8bac2ff3b4d9e0ff36a0d19e42103331d9737e8504747cd1e659f76297bd", size = 471670, upload-time = "2026-04-08T16:43:08.512Z" },
{ url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" },
{ url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" },
{ url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" },
@@ -324,9 +356,7 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" },
{ url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" },
{ url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" },
{ url = "https://files.pythonhosted.org/packages/ef/92/466b0d9afd44b8af623139a3599d651c7564fa4152f25f117e1ee5949ffb/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4cd56a9eb7a6444edbc19062f7b6fbc8f287c663b946e3171d899693b1c19fa", size = 665872, upload-time = "2026-04-08T16:40:43.912Z" },
{ url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" },
{ url = "https://files.pythonhosted.org/packages/0d/14/3395a7ef3e260de0325152ddfe19dffb3e49fe10873b94654352b53ad48e/greenlet-3.4.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:1f85f204c4d54134ae850d401fa435c89cd667d5ce9dc567571776b45941af72", size = 489237, upload-time = "2026-04-08T16:43:09.993Z" },
{ url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" },
{ url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" },
{ url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" },
@@ -341,46 +371,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" },
]
[[package]]
name = "iaai-scraper"
version = "0.1.0"
source = { editable = "." }
dependencies = [
{ name = "alembic" },
{ name = "celery" },
{ name = "fastapi" },
{ name = "playwright" },
{ name = "psycopg2-binary" },
{ name = "pydantic" },
{ name = "python-dotenv" },
{ name = "redis" },
{ name = "sqlalchemy" },
{ name = "uvicorn" },
]
[package.optional-dependencies]
dev = [
{ name = "pytest" },
{ name = "pytest-cov" },
]
[package.metadata]
requires-dist = [
{ name = "alembic", specifier = ">=1.14.0" },
{ name = "celery", specifier = ">=5.4.0" },
{ name = "fastapi", specifier = ">=0.115.0" },
{ name = "playwright", specifier = ">=1.53.0" },
{ name = "psycopg2-binary", specifier = ">=2.9.9" },
{ name = "pydantic", specifier = ">=2.8.2" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" },
{ name = "python-dotenv", specifier = ">=1.0.1" },
{ name = "redis", specifier = ">=5.2.0" },
{ name = "sqlalchemy", specifier = ">=2.0.32" },
{ name = "uvicorn", specifier = ">=0.34.0" },
]
provides-extras = ["dev"]
[[package]]
name = "idna"
version = "3.11"
@@ -509,25 +499,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" },
]
[[package]]
name = "playwright"
version = "1.58.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "greenlet" },
{ name = "pyee" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098, upload-time = "2026-01-30T15:09:24.028Z" },
{ url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625, upload-time = "2026-01-30T15:09:27.558Z" },
{ url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098, upload-time = "2026-01-30T15:09:30.461Z" },
{ url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268, upload-time = "2026-01-30T15:09:33.787Z" },
{ url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214, upload-time = "2026-01-30T15:09:36.751Z" },
{ url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998, upload-time = "2026-01-30T15:09:39.627Z" },
{ url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005, upload-time = "2026-01-30T15:09:42.449Z" },
{ url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" },
]
[[package]]
name = "pluggy"
version = "1.6.0"
@@ -713,18 +684,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" },
]
[[package]]
name = "pyee"
version = "13.0.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
]
[[package]]
name = "pygments"
version = "2.20.0"
@@ -968,6 +927,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" },
]
[[package]]
name = "urllib3"
version = "2.6.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" },
]
[[package]]
name = "uvicorn"
version = "0.44.0"