From 50d107cfbacfa4f3f3a1213c1c70a0076c1d7a28 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Wed, 8 Apr 2026 23:36:19 +0300 Subject: [PATCH] Initial project version --- .dockerignore | 16 + .env.example | 64 ++++ .gitattributes | 6 + .gitignore | 17 + Dockerfile | 22 ++ README.md | 135 ++++++++ alembic.ini | 40 +++ alembic/env.py | 58 ++++ alembic/script.py.mako | 25 ++ alembic/versions/001_initial.py | 105 +++++++ docker-compose.yml | 105 +++++++ entrypoint.sh | 57 ++++ iaai_scraper/__init__.py | 1 + iaai_scraper/api/__init__.py | 3 + iaai_scraper/api/app.py | 38 +++ iaai_scraper/api/deps.py | 7 + iaai_scraper/api/routes/__init__.py | 0 iaai_scraper/api/routes/cars.py | 137 +++++++++ iaai_scraper/api/routes/health.py | 24 ++ iaai_scraper/api/routes/tasks.py | 174 +++++++++++ iaai_scraper/browser/__init__.py | 6 + iaai_scraper/browser/factory.py | 117 +++++++ iaai_scraper/browser/listing.py | 157 ++++++++++ iaai_scraper/browser/network.py | 122 ++++++++ iaai_scraper/browser/pace.py | 46 +++ iaai_scraper/cli.py | 82 +++++ iaai_scraper/core/__init__.py | 1 + iaai_scraper/core/config.py | 137 +++++++++ iaai_scraper/core/logs.py | 31 ++ iaai_scraper/core/retry.py | 49 +++ iaai_scraper/core/utils.py | 57 ++++ iaai_scraper/parsing/__init__.py | 1 + iaai_scraper/parsing/mapper.py | 412 +++++++++++++++++++++++++ iaai_scraper/parsing/parser.py | 367 ++++++++++++++++++++++ iaai_scraper/proxy_bridge.py | 317 +++++++++++++++++++ iaai_scraper/scraper.py | 460 ++++++++++++++++++++++++++++ iaai_scraper/storage/__init__.py | 1 + iaai_scraper/storage/db.py | 216 +++++++++++++ iaai_scraper/storage/enums.py | 8 + iaai_scraper/storage/models.py | 99 ++++++ iaai_scraper/storage/schemas.py | 78 +++++ iaai_scraper/worker/__init__.py | 3 + iaai_scraper/worker/celery_app.py | 51 +++ iaai_scraper/worker/tasks.py | 133 ++++++++ main.py | 5 + pytest.ini | 4 + requirements.txt | 12 + tests/conftest.py | 7 + tests/test_db.py | 151 +++++++++ tests/test_listing.py | 42 +++ tests/test_mappers.py | 139 +++++++++ tests/test_parser.py | 62 ++++ tests/test_scraper.py | 154 ++++++++++ tests/test_utils.py | 31 ++ 54 files changed, 4592 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/001_initial.py create mode 100644 docker-compose.yml create mode 100644 entrypoint.sh create mode 100644 iaai_scraper/__init__.py create mode 100644 iaai_scraper/api/__init__.py create mode 100644 iaai_scraper/api/app.py create mode 100644 iaai_scraper/api/deps.py create mode 100644 iaai_scraper/api/routes/__init__.py create mode 100644 iaai_scraper/api/routes/cars.py create mode 100644 iaai_scraper/api/routes/health.py create mode 100644 iaai_scraper/api/routes/tasks.py create mode 100644 iaai_scraper/browser/__init__.py create mode 100644 iaai_scraper/browser/factory.py create mode 100644 iaai_scraper/browser/listing.py create mode 100644 iaai_scraper/browser/network.py create mode 100644 iaai_scraper/browser/pace.py create mode 100644 iaai_scraper/cli.py create mode 100644 iaai_scraper/core/__init__.py create mode 100644 iaai_scraper/core/config.py create mode 100644 iaai_scraper/core/logs.py create mode 100644 iaai_scraper/core/retry.py create mode 100644 iaai_scraper/core/utils.py create mode 100644 iaai_scraper/parsing/__init__.py create mode 100644 iaai_scraper/parsing/mapper.py create mode 100644 iaai_scraper/parsing/parser.py create mode 100644 iaai_scraper/proxy_bridge.py create mode 100644 iaai_scraper/scraper.py create mode 100644 iaai_scraper/storage/__init__.py create mode 100644 iaai_scraper/storage/db.py create mode 100644 iaai_scraper/storage/enums.py create mode 100644 iaai_scraper/storage/models.py create mode 100644 iaai_scraper/storage/schemas.py create mode 100644 iaai_scraper/worker/__init__.py create mode 100644 iaai_scraper/worker/celery_app.py create mode 100644 iaai_scraper/worker/tasks.py create mode 100644 main.py create mode 100644 pytest.ini create mode 100644 requirements.txt create mode 100644 tests/conftest.py create mode 100644 tests/test_db.py create mode 100644 tests/test_listing.py create mode 100644 tests/test_mappers.py create mode 100644 tests/test_parser.py create mode 100644 tests/test_scraper.py create mode 100644 tests/test_utils.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..acf045b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +__pycache__/ +.pytest_cache/ +.coverage +coverage.xml +.venv/ +*.pyc +*.pyo +*.pyd +*.log +*.db +.env +.git/ +.vscode/ +*.egg-info/ +dist/ +build/ \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3b193ec --- /dev/null +++ b/.env.example @@ -0,0 +1,64 @@ +IAAI_HEADLESS=true +IAAI_LOG_LEVEL=INFO +# IAAI_LOG_FILE=iaai_scraper.log + +# Network capture settings. +IAAI_CAPTURE_SAME_ORIGIN_ONLY=true +IAAI_MAX_CAPTURED_REQUESTS=40 +IAAI_MAX_CAPTURED_JSON_RESPONSES=30 + +# Sequential listing-first mode. +IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars +IAAI_MAX_PAGES_PER_RUN=10 +IAAI_MAX_VEHICLES_PER_RUN=30 +IAAI_PAGE_LINK_LIMIT=100 +IAAI_INCLUDE_PAGINATION=true +IAAI_COLLECT_CURRENT_PAGE_ONLY=false + +# Human-like pacing. +IAAI_HUMAN_PACE_ENABLED=true +IAAI_AFTER_LISTING_OPEN_MIN_S=2.5 +IAAI_AFTER_LISTING_OPEN_MAX_S=4.5 +IAAI_AFTER_FILTER_ACTION_MIN_S=2.0 +IAAI_AFTER_FILTER_ACTION_MAX_S=4.0 +IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5 +IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0 +IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0 +IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5 +IAAI_BETWEEN_VEHICLES_MIN_S=3.0 +IAAI_BETWEEN_VEHICLES_MAX_S=6.0 +IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0 +IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0 + +# Sync settings +IAAI_SYNC_ONLY_NEW=true + +# Retry / backoff +IAAI_RETRY_DELAY_SECONDS=2.5 +IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 +IAAI_RETRY_JITTER_SECONDS=0.25 + +# Proxy (HTTP/HTTPS preferred for Playwright) +# Chromium does not support SOCKS5 proxy authentication directly. +# Use residential or mobile USA proxy. +# IAAI_PROXY_SERVER=http://proxy.example.com:8080 +# IAAI_PROXY_USERNAME= +# IAAI_PROXY_PASSWORD= + +# Database (PostgreSQL). +IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper +IAAI_DATABASE_ECHO=false +IAAI_DATABASE_POOL_SIZE=5 +IAAI_DATABASE_MAX_OVERFLOW=10 + +# ─── Redis (Celery broker) ───────────────────────────────── +IAAI_REDIS_URL=redis://redis:6379/0 + +# ─── Celery ──────────────────────────────────────────────── +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 +CELERY_TASK_SOFT_TIME_LIMIT=600 +CELERY_TASK_TIME_LIMIT=900 +CELERY_WORKER_CONCURRENCY=1 +CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 +CELERY_BEAT_SYNC_LIMIT=26 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b06c566 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Force LF for shell scripts (critical for Docker on Windows) +*.sh text eol=lf +entrypoint.sh text eol=lf + +# Default for other files +* text=auto diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7755897 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +__pycache__/ +.pytest_cache/ +.coverage +coverage.xml +.venv/ +*.pyc +*.pyo +*.pyd +*.log +*.db +.env +.vscode/ +*.egg-info/ +dist/ +build/ +artifacts/ +celerybeat-schedule* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f3df83d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM mcr.microsoft.com/playwright/python:v1.58.0-noble + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +# Xvfb for headed Chromium in container (IAAI blocks headless) +RUN apt-get update -qq && apt-get install -y --no-install-recommends xvfb \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . +RUN chmod +x entrypoint.sh + +STOPSIGNAL SIGINT + +# По умолчанию — API, но worker/beat переопределяют CMD в docker-compose +ENTRYPOINT ["./entrypoint.sh"] +CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..fa8a77e --- /dev/null +++ b/README.md @@ -0,0 +1,135 @@ +# IAAI Scraper + +Небольшой сервис для сбора автомобилей с [iaai.com](https://www.iaai.com). +Собирает ссылки из листинга, открывает карточки машин, достаёт данные из HTML и XHR, сохраняет всё в PostgreSQL. + +## Что важно по сайту + +У IAAI стоит **Imperva Incapsula** — внешний anti-bot и WAF. + +- headless Chromium без прокси часто режется +- часть данных грузится через XHR, часть есть сразу в HTML +- есть cookie banner + +Поэтому в проекте используется Playwright, паузы между действиями и прокси. + + +## Запуск + +```bash +cp .env.example .env # подправить под себя +docker compose up -d +``` + +Поднимутся 5 контейнеров: postgres, redis, api, worker, beat. API на `http://localhost:8000`. + +Swagger-документация: `http://localhost:8000/docs` + +## API + +**Здоровье и статистика:** +- `GET /health` — статус сервиса и подключения к БД +- `GET /api/v1/stats` — сколько машин/картинок в базе, топ брендов + +**Машины:** +- `GET /api/v1/cars` — список с пагинацией +- `GET /api/v1/cars/{id}` — карточка с картинками +- `GET /api/v1/cars/by-origin/{origin_id}` — поиск по IAAI stock number + +**Задачи:** +- `POST /api/v1/tasks/sync-vehicle` — скрапнуть одну машину по URL +- `POST /api/v1/tasks/sync-listing` — запустить полный обход листинга +- `GET /api/v1/tasks/{task_id}` — статус задачи +- `GET /api/v1/tasks` — все задачи +- `GET /api/v1/sync-runs` — история запусков + +Пример: +```bash +curl -X POST http://localhost:8000/api/v1/tasks/sync-vehicle + -H "Content-Type: application/json" \ + -d '{"vehicle_url": "https://www.iaai.com/VehicleDetail/45089484~US"}' +``` + +## CLI + +Для локального запуска без API и Celery: +```bash +python main.py init-db +python main.py collect-listing --make Toyota +python main.py sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US" +python main.py sync-listing --limit 10 +``` + +## Структура + +``` +iaai_scraper/ + scraper.py — основной код скрапера + cli.py — CLI-команды + proxy_bridge.py — HTTP→SOCKS5 мост + + browser/ + factory.py — запуск браузера и контекста + listing.py — сбор ссылок из листинга + network.py — перехват XHR/fetch + pace.py — паузы между действиями + + parsing/ + parser.py — разбор карточки машины + mapper.py — нормализация данных + + storage/ + models.py — SQLAlchemy модели + schemas.py — Pydantic схемы + enums.py — справочники значений + db.py — работа с БД + + api/ + app.py — FastAPI приложение + deps.py — зависимости + routes/ + health.py — GET /health + cars.py — машины и статистика + tasks.py — задачи и история запусков + + worker/ + celery_app.py — конфиг Celery + tasks.py — фоновые задачи + + core/ + config.py — настройки + logs.py — логирование + retry.py — retry-логика + utils.py — утилиты + +alembic/ — миграции БД +``` + +## Конфигурация + +Основные переменные лежат в `.env.example`: + +**БД:** `IAAI_DATABASE_URL`, `IAAI_DATABASE_POOL_SIZE` +**Redis:** `IAAI_REDIS_URL` +**Celery:** `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES` +**Скрапер:** `IAAI_HEADLESS`, `IAAI_SYNC_ONLY_NEW`, `IAAI_MAX_PAGES_PER_RUN` +**Прокси:** `IAAI_PROXY_SERVER`, `IAAI_PROXY_USERNAME`, `IAAI_PROXY_PASSWORD` +**Паузы:** `IAAI_BETWEEN_VEHICLES_MIN_S`, `IAAI_AFTER_PAGE_CHANGE_MAX_S` и т.д. + +## Миграции + +В Docker миграции запускаются автоматически при старте API-контейнера. + +Вручную: +```bash +alembic upgrade head +alembic revision --autogenerate -m "add_column_x" +``` + +## Тесты + +```bash +pytest -q +``` + +Тесты идут на SQLite in-memory, без внешних сервисов. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..f4be675 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +# Alembic Configuration File + +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..03ffa6e --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,58 @@ +"""Alembic env.py — подключение к БД через Settings.""" + +import os +import sys +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +# Добавляем корень проекта в sys.path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from iaai_scraper.core.config import Settings +from iaai_scraper.storage.models import Base + +config = context.config + +# Logging +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Подставляем URL из Settings (env vars имеют приоритет) +settings = Settings() +config.set_main_option("sqlalchemy.url", settings.database.url) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode.""" + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..958df87 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py new file mode 100644 index 0000000..d9b2c69 --- /dev/null +++ b/alembic/versions/001_initial.py @@ -0,0 +1,105 @@ +"""Initial schema — cars, images, sync_runs, scrape_tasks + +Revision ID: 001_initial +Revises: +Create Date: 2026-04-08 +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # --- cars --- + op.create_table( + "cars", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("parser_id", sa.String(50), nullable=False, unique=True), + sa.Column("brand", sa.String(50), nullable=False), + sa.Column("model", sa.String(50), nullable=False), + sa.Column("year", sa.Integer, nullable=True), + sa.Column("price", sa.BigInteger, nullable=True), + sa.Column("currency", sa.String(10), nullable=False, server_default="USD"), + sa.Column("mileage", sa.Integer, nullable=False, server_default="0"), + sa.Column("country", sa.String(10), nullable=False, server_default="NA"), + sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("color", sa.String, nullable=False, server_default="other"), + sa.Column("drive", sa.String(10), nullable=True), + sa.Column("gearbox", sa.String(10), nullable=True), + sa.Column("steering_wheel", sa.String(10), nullable=True), + sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"), + sa.Column("engine_volume", sa.Integer, nullable=True), + sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"), + sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("origin", sa.String(20), nullable=False, server_default="NA"), + sa.Column("origin_url", sa.String, nullable=False), + sa.Column("origin_id", sa.String, nullable=False, unique=True), + sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("evaluation", sa.String, nullable=True), + sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")), + sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("slug", sa.String, nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("content_hash", sa.String(64), nullable=False, server_default=""), + sa.Column("raw_attributes", sa.Text, nullable=True), + ) + op.create_index("ix_cars_origin_url", "cars", ["origin_url"]) + op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True) + op.create_index("ix_cars_content_hash", "cars", ["content_hash"]) + + # --- images --- + op.create_table( + "images", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("fullres_image", sa.String, nullable=False), + sa.Column("preview_image", sa.String, nullable=False), + sa.Column("order_index", sa.Integer, nullable=False), + sa.Column("car_id", sa.BigInteger, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False), + ) + + # --- sync_runs --- + op.create_table( + "sync_runs", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("status", sa.Text, nullable=False), + sa.Column("lane", sa.Text, nullable=False), + sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"), + sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("error_summary", sa.Text, nullable=True), + ) + + # --- scrape_tasks --- + op.create_table( + "scrape_tasks", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("celery_task_id", sa.String(255), nullable=False, unique=True), + sa.Column("task_type", sa.String(50), nullable=False), + sa.Column("status", sa.String(20), nullable=False, server_default="pending"), + sa.Column("vehicle_url", sa.Text, nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("result_summary", sa.Text, nullable=True), + sa.Column("error_message", sa.Text, nullable=True), + ) + op.create_index("ix_scrape_tasks_celery_task_id", "scrape_tasks", ["celery_task_id"], unique=True) + + +def downgrade() -> None: + op.drop_table("scrape_tasks") + op.drop_table("sync_runs") + op.drop_table("images") + op.drop_table("cars") diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..750c37f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,105 @@ +services: + # ─── PostgreSQL ─────────────────────────────────────────── + postgres: + image: postgres:16-alpine + container_name: iaai-postgres + restart: unless-stopped + environment: + POSTGRES_USER: iaai + POSTGRES_PASSWORD: iaai + POSTGRES_DB: iaai_scraper + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"] + interval: 5s + timeout: 3s + retries: 5 + + # ─── Redis (Celery broker) ──────────────────────────────── + redis: + image: redis:7-alpine + container_name: iaai-redis + restart: unless-stopped + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + # ─── FastAPI ────────────────────────────────────────────── + api: + build: . + container_name: iaai-api + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper + IAAI_REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ports: + - "8000:8000" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + command: > + uvicorn iaai_scraper.api.app:app + --host 0.0.0.0 --port 8000 --workers 2 + + # ─── Celery Worker ──────────────────────────────────────── + worker: + build: . + container_name: iaai-worker + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper + IAAI_REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + stop_grace_period: 60s + command: > + celery -A iaai_scraper.worker.celery_app worker + --loglevel=info --concurrency=1 --pool=prefork + -Q scraping --without-heartbeat + + # ─── Celery Beat (периодический планировщик) ────────────── + beat: + build: . + container_name: iaai-beat + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper + IAAI_REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + command: > + celery -A iaai_scraper.worker.celery_app beat + --loglevel=info + +volumes: + pgdata: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..869b0cc --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -e + +# Start HTTP→SOCKS5 proxy bridge if SOCKS5 upstream is configured +if [ -n "$SOCKS5_PROXY_HOST" ]; then + echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 $SOCKS5_PROXY_HOST:${SOCKS5_PROXY_PORT:-1002})..." + if [ -z "$IAAI_PROXY_SERVER" ]; then + export IAAI_PROXY_SERVER="http://127.0.0.1:8899" + elif [ "$IAAI_PROXY_SERVER" = "http://localhost:8899" ]; then + export IAAI_PROXY_SERVER="http://127.0.0.1:8899" + fi + echo "[entrypoint] Using browser proxy: $IAAI_PROXY_SERVER" + python -m iaai_scraper.proxy_bridge & + BRIDGE_PID=$! + sleep 1 + if ! kill -0 $BRIDGE_PID 2>/dev/null; then + echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start" + exit 1 + fi + echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)" +fi + +# Xvfb needed only for worker (Playwright) — not for API or beat +NEEDS_XVFB=false +case "$1" in + celery*|python*main*) + NEEDS_XVFB=true + ;; + *) + # Check if any arg contains "worker" + for arg in "$@"; do + case "$arg" in + *worker*) NEEDS_XVFB=true; break ;; + esac + done + ;; +esac + +if [ "$NEEDS_XVFB" = "true" ]; then + # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display + export DISPLAY=:99 + export IAAI_HEADLESS=false + Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp & + XVFB_PID=$! + sleep 0.5 + echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)" +fi + +# Run Alembic migrations (only for API service, skip for worker/beat) +case "$1" in + uvicorn*) + echo "[entrypoint] Running Alembic migrations..." + alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..." + ;; +esac + +exec "$@" diff --git a/iaai_scraper/__init__.py b/iaai_scraper/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/api/__init__.py b/iaai_scraper/api/__init__.py new file mode 100644 index 0000000..b94a1e8 --- /dev/null +++ b/iaai_scraper/api/__init__.py @@ -0,0 +1,3 @@ +from .app import create_app + +__all__ = ["create_app"] diff --git a/iaai_scraper/api/app.py b/iaai_scraper/api/app.py new file mode 100644 index 0000000..4aa00fb --- /dev/null +++ b/iaai_scraper/api/app.py @@ -0,0 +1,38 @@ +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from ..core.config import Settings +from ..storage.db import PersistenceService +from .routes import cars, health, tasks + + +@asynccontextmanager +async def lifespan(app: FastAPI): + persistence: PersistenceService = app.state.persistence + persistence.create_tables() + yield + + +def create_app(settings: Settings | None = None) -> FastAPI: + _settings = settings or Settings() + + app = FastAPI( + title="IAAI Scraper API", + description="REST API для управления задачами скрапинга IAAI и просмотра данных", + version="1.0.0", + lifespan=lifespan, + ) + + app.state.settings = _settings + app.state.persistence = PersistenceService(_settings) + + app.include_router(health.router, tags=["health"]) + app.include_router(cars.router, prefix="/api/v1", tags=["cars"]) + app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"]) + + return app + + +# uvicorn entrypoint +app = create_app() diff --git a/iaai_scraper/api/deps.py b/iaai_scraper/api/deps.py new file mode 100644 index 0000000..a642b12 --- /dev/null +++ b/iaai_scraper/api/deps.py @@ -0,0 +1,7 @@ +from fastapi import Request + +from ..storage.db import PersistenceService + + +def get_persistence(request: Request) -> PersistenceService: + return request.app.state.persistence diff --git a/iaai_scraper/api/routes/__init__.py b/iaai_scraper/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/iaai_scraper/api/routes/cars.py b/iaai_scraper/api/routes/cars.py new file mode 100644 index 0000000..e61cd8b --- /dev/null +++ b/iaai_scraper/api/routes/cars.py @@ -0,0 +1,137 @@ +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import Car, Image + +router = APIRouter() + + +@router.get("/cars") +def list_cars( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + brand: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + is_sold: bool | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + query = select(Car) + + if brand: + query = query.where(Car.brand.ilike(f"%{brand}%")) + if model: + query = query.where(Car.model.ilike(f"%{model}%")) + if year_min is not None: + query = query.where(Car.year >= year_min) + if year_max is not None: + query = query.where(Car.year <= year_max) + if is_sold is not None: + query = query.where(Car.is_sold == is_sold) + + # Общее количество. + count_query = select(func.count()).select_from(query.subquery()) + total = session.execute(count_query).scalar() or 0 + + # пагинация + offset = (page - 1) * per_page + cars = session.execute( + query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if per_page else 0, + "items": [_car_to_dict(car) for car in cars], + } + + +@router.get("/cars/{car_id}") +def get_car( + car_id: int, + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + car = session.get(Car, car_id) + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return _car_to_dict(car, include_images=True) + + +@router.get("/cars/by-origin/{origin_id}") +def get_car_by_origin( + origin_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + car = session.execute( + select(Car).where(Car.origin_id == origin_id) + ).scalars().first() + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return _car_to_dict(car, include_images=True) + + +@router.get("/stats") +def get_stats(persistence: PersistenceService = Depends(get_persistence)): + with persistence.session_scope() as session: + total_cars = session.execute(select(func.count(Car.id))).scalar() or 0 + total_images = session.execute(select(func.count(Image.id))).scalar() or 0 + brands = session.execute( + select(Car.brand, func.count(Car.id)) + .group_by(Car.brand) + .order_by(func.count(Car.id).desc()) + .limit(20) + ).all() + + return { + "total_cars": total_cars, + "total_images": total_images, + "top_brands": [{"brand": b, "count": c} for b, c in brands], + } + + +def _car_to_dict(car: Car, include_images: bool = False) -> dict: + result = { + "id": car.id, + "parser_id": car.parser_id, + "brand": car.brand, + "model": car.model, + "year": car.year, + "price": car.price, + "currency": car.currency, + "mileage": car.mileage, + "country": car.country, + "is_sold": car.is_sold, + "color": car.color, + "drive": car.drive, + "gearbox": car.gearbox, + "steering_wheel": car.steering_wheel, + "body_type": car.body_type, + "engine_volume": car.engine_volume, + "selling_type": car.selling_type, + "origin": car.origin, + "origin_url": car.origin_url, + "origin_id": car.origin_id, + "is_damaged": car.is_damaged, + "slug": car.slug, + "last_seen_at": car.last_seen_at.isoformat() if car.last_seen_at else None, + "content_hash": car.content_hash, + } + if include_images: + result["images"] = [ + { + "id": img.id, + "fullres_image": img.fullres_image, + "preview_image": img.preview_image, + "order_index": img.order_index, + } + for img in sorted(car.images, key=lambda i: i.order_index) + ] + return result diff --git a/iaai_scraper/api/routes/health.py b/iaai_scraper/api/routes/health.py new file mode 100644 index 0000000..1fdfc40 --- /dev/null +++ b/iaai_scraper/api/routes/health.py @@ -0,0 +1,24 @@ +from fastapi import APIRouter, Depends +from sqlalchemy import text + +from ..deps import get_persistence +from ...storage.db import PersistenceService + +router = APIRouter() + + +@router.get("/health") +def health_check(persistence: PersistenceService = Depends(get_persistence)): + db_ok = False + try: + with persistence.session_scope() as session: + session.execute(text("SELECT 1")) + db_ok = True + except Exception: + pass + + return { + "status": "ok" if db_ok else "degraded", + "service": "iaai-scraper-api", + "database": "connected" if db_ok else "unavailable", + } diff --git a/iaai_scraper/api/routes/tasks.py b/iaai_scraper/api/routes/tasks.py new file mode 100644 index 0000000..9071c3b --- /dev/null +++ b/iaai_scraper/api/routes/tasks.py @@ -0,0 +1,174 @@ +from datetime import datetime, timezone + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel +from sqlalchemy import select, func + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import ScrapeTask, SyncRun +from ...worker.tasks import sync_vehicle_task, sync_listing_task + +router = APIRouter() + + +# --- схемы + +class SyncVehicleRequest(BaseModel): + vehicle_url: str + lane: str = "iaai" + + +class SyncListingRequest(BaseModel): + make: str | None = None + model: str | None = None + lane: str = "iaai_cars" + limit: int | None = None + only_new: bool | None = None + + +# --- эндпоинты + +@router.post("/tasks/sync-vehicle") +def start_sync_vehicle( + body: SyncVehicleRequest, + persistence: PersistenceService = Depends(get_persistence), +): + result = sync_vehicle_task.apply_async( + args=(body.vehicle_url,), + kwargs={"lane": body.lane}, + queue="scraping", + ) + + # регистрируем в бд + persistence.create_scrape_task( + celery_task_id=result.id, + task_type="sync_vehicle", + vehicle_url=body.vehicle_url, + ) + + return { + "task_id": result.id, + "status": "queued", + "vehicle_url": body.vehicle_url, + } + + +@router.post("/tasks/sync-listing") +def start_sync_listing( + body: SyncListingRequest, + persistence: PersistenceService = Depends(get_persistence), +): + result = sync_listing_task.apply_async( + kwargs={ + "make": body.make, + "model": body.model, + "lane": body.lane, + "limit": body.limit, + "only_new": body.only_new, + }, + queue="scraping", + ) + + persistence.create_scrape_task( + celery_task_id=result.id, + task_type="sync_listing", + ) + + return { + "task_id": result.id, + "status": "queued", + } + + +@router.get("/tasks/{task_id}") +def get_task_status( + task_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + task_info = persistence.get_scrape_task(task_id) + if task_info is None: + raise HTTPException(status_code=404, detail="Task not found") + return task_info + + +@router.get("/tasks") +def list_tasks( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + status: str | None = None, + task_type: str | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + query = select(ScrapeTask) + if status: + query = query.where(ScrapeTask.status == status) + if task_type: + query = query.where(ScrapeTask.task_type == task_type) + + total = session.execute( + select(func.count()).select_from(query.subquery()) + ).scalar() or 0 + + offset = (page - 1) * per_page + tasks = session.execute( + query.order_by(ScrapeTask.created_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": t.id, + "celery_task_id": t.celery_task_id, + "task_type": t.task_type, + "status": t.status, + "vehicle_url": t.vehicle_url, + "created_at": t.created_at.isoformat() if t.created_at else None, + "started_at": t.started_at.isoformat() if t.started_at else None, + "finished_at": t.finished_at.isoformat() if t.finished_at else None, + "result_summary": t.result_summary, + "error_message": t.error_message, + } + for t in tasks + ], + } + + +@router.get("/sync-runs") +def list_sync_runs( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 + + offset = (page - 1) * per_page + runs = session.execute( + select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": r.id, + "started_at": r.started_at.isoformat() if r.started_at else None, + "finished_at": r.finished_at.isoformat() if r.finished_at else None, + "status": r.status, + "lane": r.lane, + "ids_fetched": r.ids_fetched, + "cars_upserted": r.cars_upserted, + "cars_failed": r.cars_failed, + "images_upserted": r.images_upserted, + "error_summary": r.error_summary, + } + for r in runs + ], + } diff --git a/iaai_scraper/browser/__init__.py b/iaai_scraper/browser/__init__.py new file mode 100644 index 0000000..38bb3f4 --- /dev/null +++ b/iaai_scraper/browser/__init__.py @@ -0,0 +1,6 @@ +from .factory import BrowserFactory +from .listing import ListingCollector +from .network import NetworkCapture +from .pace import HumanPacer + +__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"] diff --git a/iaai_scraper/browser/factory.py b/iaai_scraper/browser/factory.py new file mode 100644 index 0000000..0759eb1 --- /dev/null +++ b/iaai_scraper/browser/factory.py @@ -0,0 +1,117 @@ +import json +import logging +import random + +from playwright.sync_api import Browser, BrowserContext, Playwright + +from ..core.config import Settings + +logger = logging.getLogger("iaai_scraper.browser") + + +def _build_init_script() -> str: + """JS-патч признаков автоматизации.""" + hardware_concurrency = random.choice([4, 8, 12, 16]) + device_memory = random.choice([4, 8, 16]) + languages = ["en-US", "en"] + + return f""" +(() => {{ + const define = (obj, prop, value) => {{ + try {{ + Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); + }} catch (e) {{}} + }}; + + define(navigator, 'webdriver', undefined); + define(navigator, 'platform', 'Win32'); + define(navigator, 'vendor', 'Google Inc.'); + define(navigator, 'language', '{languages[0]}'); + define(navigator, 'languages', {json.dumps(languages)}); + define(navigator, 'hardwareConcurrency', {hardware_concurrency}); + define(navigator, 'deviceMemory', {device_memory}); + define(navigator, 'maxTouchPoints', 0); + + if (!window.chrome) {{ + Object.defineProperty(window, 'chrome', {{ + value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, + configurable: true + }}); + }} + + const originalQuery = navigator.permissions && navigator.permissions.query; + if (originalQuery) {{ + navigator.permissions.query = (params) => ( + params && params.name === 'notifications' + ? Promise.resolve({{ state: Notification.permission }}) + : originalQuery(params) + ); + }} + + const originalGetParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(parameter) {{ + if (parameter === 37445) return 'Intel Inc.'; + if (parameter === 37446) return 'Intel Iris OpenGL Engine'; + return originalGetParameter.call(this, parameter); + }}; +}})(); +""" + + +class BrowserFactory: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def create_browser(self, playwright: Playwright) -> Browser: + launch_kwargs: dict = { + "headless": self.settings.headless, + "args": [ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-features=IsolateOrigins,site-per-process", + ], + } + proxy_dict = self.settings.proxy.to_playwright_dict() + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + try: + logger.info("Trying to launch real Chrome channel") + return playwright.chromium.launch(channel="chrome", **launch_kwargs) + except Exception: + logger.warning("Chrome channel launch failed, falling back to Chromium") + return playwright.chromium.launch(**launch_kwargs) + + def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext: + viewport = random.choice(self.settings.fingerprint.viewport_presets) + timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) + color_scheme = random.choice(["light", "dark"]) + + context = browser.new_context( + storage_state=storage_state or None, + user_agent=self.settings.fingerprint.user_agent, + viewport=viewport, + screen=viewport, + device_scale_factor=random.choice([1, 1.25]), + is_mobile=False, + has_touch=False, + locale=self.settings.fingerprint.locale, + timezone_id=timezone_id, + color_scheme=color_scheme, + java_script_enabled=True, + ignore_https_errors=False, + extra_http_headers={ + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', + }, + ) + context.set_default_timeout(self.settings.default_timeout_ms) + context.set_default_navigation_timeout(self.settings.default_timeout_ms) + context.add_init_script(_build_init_script()) + return context diff --git a/iaai_scraper/browser/listing.py b/iaai_scraper/browser/listing.py new file mode 100644 index 0000000..853088e --- /dev/null +++ b/iaai_scraper/browser/listing.py @@ -0,0 +1,157 @@ +import logging +import re +from dataclasses import asdict, dataclass, field +from typing import Any +from urllib.parse import urljoin + +from playwright.sync_api import Page + +from .pace import HumanPacer +from ..core.config import Settings +from ..core.utils import first_non_empty + +logger = logging.getLogger("iaai_scraper.listing") +VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE) + + +@dataclass(slots=True) +class ListingVehicleLink: + # Ссылка на карточку + href: str + title: str = "" + lot_number: str | None = None + + +@dataclass(slots=True) +class ListingPageResult: + # Результат страницы листинга + source_url: str + page_number: int + vehicle_links: list[ListingVehicleLink] = field(default_factory=list) + pagination_available: bool = False + next_page_detected: bool = False + + +class ListingCollector: + def __init__(self, settings: Settings, pacer: HumanPacer) -> None: + # Сборщик ссылок листинга + self.settings = settings + self.pacer = pacer + + def open_cars_listing(self, page: Page) -> None: + # Открываем листинг и ждём ссылки + logger.info("Opening cars listing page: %s", self.settings.listing.cars_url) + page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded") + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + logger.debug("networkidle timeout on listing page, continuing with current state") + try: + page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000) + except Exception: + logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully") + self.pacer.after_listing_open() + + def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]: + # Пробуем make/model фильтры + applied = {"make": None, "model": None} + if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): + applied["make"] = make + self.pacer.after_filter_action() + if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): + applied["model"] = model + self.pacer.after_filter_action() + return applied + + def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: + # Собираем ссылки текущей страницы + anchors = page.locator("a[href*='/VehicleDetail/']") + total = min(anchors.count(), self.settings.listing.page_link_limit) + links: list[ListingVehicleLink] = [] + seen: set[str] = set() + for idx in range(total): + anchor = anchors.nth(idx) + href = anchor.get_attribute("href") or "" + match = VEHICLE_HREF_RE.search(href) + if not match: + continue + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or "" + links.append(ListingVehicleLink(href=absolute, title=str(title).strip())) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + next_page_detected = self._has_next_page(page) + return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) + + def go_to_next_page(self, page: Page) -> bool: + # Переход на следующую страницу + selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"] + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + disabled = (locator.get_attribute("disabled") or "").lower() + aria_disabled = (locator.get_attribute("aria-disabled") or "").lower() + classes = (locator.get_attribute("class") or "").lower() + if disabled or aria_disabled == "true" or "disabled" in classes: + continue + self.pacer.move_mouse_to(page, locator) + locator.click() + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass + self.pacer.after_page_change() + return True + return False + + def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]: + # Последовательный сбор с лимитами + self.open_cars_listing(page) + applied_filters = self.apply_filters(page, make=make, model=model) + pages: list[dict[str, object]] = [] + all_links: list[str] = [] + for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): + page_result = self.collect_current_page(page, page_number=page_number) + pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected}) + for item in page_result.vehicle_links: + if item.href not in all_links: + all_links.append(item.href) + if len(all_links) >= self.settings.listing.max_vehicles_per_run: + break + if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected: + break + if not self.go_to_next_page(page): + break + return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}} + + @staticmethod + def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: + # Несколько селекторов для фильтра + for selector in selectors: + locator = page.locator(selector).first + if locator.count() == 0: + continue + try: + locator.click() + locator.fill(value) + page.keyboard.press("Enter") + try: + page.wait_for_load_state("networkidle", timeout=15000) + except Exception: + pass + return True + except Exception: + continue + return False + + @staticmethod + def _has_next_page(page: Page) -> bool: + # Проверка кнопки next + for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]: + if page.locator(selector).count() > 0: + return True + return False diff --git a/iaai_scraper/browser/network.py b/iaai_scraper/browser/network.py new file mode 100644 index 0000000..7dd5f89 --- /dev/null +++ b/iaai_scraper/browser/network.py @@ -0,0 +1,122 @@ +import json +import logging +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urlparse + +from playwright.sync_api import Page, Request, Response + +from ..core.config import Settings + +logger = logging.getLogger("iaai_scraper.network") + + +@dataclass +class NetworkCapture: + """XHR/fetch перехватчик.""" + + settings: Settings + requests: list[dict[str, Any]] = field(default_factory=list) + json_responses: list[dict[str, Any]] = field(default_factory=list) + _seen_req: set[str] = field(default_factory=set) + _seen_resp: set[str] = field(default_factory=set) + _origin: str | None = None + + def attach(self, page: Page, origin_url: str | None = None) -> None: + # Подписка на сетевые события + try: + self._origin = urlparse(origin_url or page.url).netloc.lower() or None + except Exception: + self._origin = None + page.on("request", self._on_request) + page.on("response", self._on_response) + + def _is_same_origin(self, url: str) -> bool: + # Фильтр по домену + if not self.settings.capture.capture_same_origin_only or not self._origin: + return True + netloc = urlparse(url).netloc.lower() + return netloc == self._origin or netloc.endswith(".iaai.com") + + def _on_request(self, request: Request) -> None: + # Берём только xhr/fetch + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(request.url): + return + if len(self.requests) >= self.settings.capture.max_requests: + return + key = f"{request.method}:{request.url}:{request.post_data or ''}" + # Убираем дубли + if key in self._seen_req: + return + self._seen_req.add(key) + self.requests.append({ + "url": request.url, "method": request.method, + "resource_type": request.resource_type, "post_data": request.post_data, + }) + + def _on_response(self, response: Response) -> None: + # Сохраняем JSON + request = response.request + if request.resource_type not in {"xhr", "fetch"}: + return + if not self._is_same_origin(response.url): + return + if len(self.json_responses) >= self.settings.capture.max_json_responses: + return + if not self._is_json(response): + return + key = f"{request.method}:{response.url}:{response.status}" + if key in self._seen_resp: + return + self._seen_resp.add(key) + try: + payload = response.json() + except Exception: + try: + payload = json.loads(response.text()) + except Exception: + return + self.json_responses.append({ + "url": response.url, "status": response.status, + "request_method": request.method, "post_data": request.post_data, + "resource_type": request.resource_type, "payload": payload, + "category": self._categorize(response.url), + }) + + @staticmethod + def _is_json(response: Response) -> bool: + ct = (response.headers.get("content-type") or "").lower() + if "application/json" in ct or "+json" in ct: + return True + url = response.url.lower() + return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) + + @staticmethod + def _categorize(url: str) -> str: + # Простая категория URL + low = url.lower() + mapping = { + "images": ["image", "media", "photos", "gallery"], + "bids": ["bid", "offer", "buy-now", "buynow"], + "auction": ["auction", "sale", "lane", "branch"], + "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], + "documents": ["title", "document", "report"], + } + for cat, markers in mapping.items(): + if any(m in low for m in markers): + return cat + return "other" + + def export(self) -> dict[str, Any]: + responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) + return { + "requests": self.requests, + "json_responses": responses, + "capture_limits": { + "same_origin_only": self.settings.capture.capture_same_origin_only, + "max_requests": self.settings.capture.max_requests, + "max_json_responses": self.settings.capture.max_json_responses, + }, + } diff --git a/iaai_scraper/browser/pace.py b/iaai_scraper/browser/pace.py new file mode 100644 index 0000000..418dbab --- /dev/null +++ b/iaai_scraper/browser/pace.py @@ -0,0 +1,46 @@ +import random +import time + +from playwright.sync_api import Locator, Page + +from ..core.config import Settings + + +class HumanPacer: + """Random паузы между действиями.""" + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def pause(self, min_s: float, max_s: float) -> None: + if self.settings.pace.enabled: + time.sleep(random.uniform(min_s, max_s)) + + def after_listing_open(self) -> None: + self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) + + def after_filter_action(self) -> None: + self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) + + def before_vehicle_open(self) -> None: + self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) + + def after_vehicle_open(self) -> None: + self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) + + def between_vehicles(self) -> None: + self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) + + def after_page_change(self) -> None: + self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) + + def move_mouse_to(self, page: Page, locator: Locator) -> None: + try: + box = locator.bounding_box() + except Exception: + box = None + if not box: + return + x = box["x"] + box["width"] * random.uniform(0.2, 0.8) + y = box["y"] + box["height"] * random.uniform(0.2, 0.8) + page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py new file mode 100644 index 0000000..c3b76d5 --- /dev/null +++ b/iaai_scraper/cli.py @@ -0,0 +1,82 @@ +import argparse +from pathlib import Path + +from .core.config import Settings +from .core.utils import save_to_json +from .scraper import IAAIScraper + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="IAAI scraper CLI") + parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") + parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") + subparsers = parser.add_subparsers(dest="command", required=True) + + default_output_dir = Path("artifacts/json") + + subparsers.add_parser("init-db", help="Create DB tables") + + listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page") + listing_parser.add_argument("--make", default=None) + listing_parser.add_argument("--model", default=None) + listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json")) + + scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page") + scrape_parser.add_argument("vehicle_url") + scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json")) + + sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle") + sync_vehicle_parser.add_argument("vehicle_url") + sync_vehicle_parser.add_argument("--lane", default="iaai") + sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json")) + + sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles") + sync_listing_parser.add_argument("--make", default=None) + sync_listing_parser.add_argument("--model", default=None) + sync_listing_parser.add_argument("--lane", default="iaai_cars") + sync_listing_parser.add_argument("--limit", type=int, default=None) + sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) + sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json")) + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + runtime_settings: Settings | None = None + if args.headless is not None or args.debug: + runtime_settings = Settings() + if args.headless is not None: + runtime_settings.headless = args.headless == "true" + if args.debug: + runtime_settings.log_level = "DEBUG" + + with IAAIScraper(runtime_settings) as scraper: + if args.command == "init-db": + data = scraper.init_db() + print(f"DB initialized: {data}") + return + elif args.command == "collect-listing": + data = scraper.collect_listing(make=args.make, model=args.model) + elif args.command == "scrape-vehicle": + data = scraper.scrape_vehicle_detail(args.vehicle_url) + elif args.command == "sync-vehicle": + data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) + else: + only_new = None if args.only_new is None else args.only_new == "true" + data = scraper.sync_listing( + make=args.make, + model=args.model, + lane=args.lane, + limit=args.limit, + only_new=only_new, + ) + + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + + +if __name__ == "__main__": + main() diff --git a/iaai_scraper/core/__init__.py b/iaai_scraper/core/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/core/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py new file mode 100644 index 0000000..725194d --- /dev/null +++ b/iaai_scraper/core/config.py @@ -0,0 +1,137 @@ +import os +from dataclasses import dataclass, field + +from dotenv import load_dotenv + +load_dotenv() + + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +@dataclass(slots=True) +class CaptureConfig: + capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"} + max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40")) + max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30")) + + +@dataclass(slots=True) +class HumanPaceConfig: + enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"} + after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5")) + after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5")) + after_filter_action_min_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MIN_S", "2.0")) + after_filter_action_max_s: float = float(os.getenv("IAAI_AFTER_FILTER_ACTION_MAX_S", "4.0")) + before_vehicle_open_min_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", "0.5")) + before_vehicle_open_max_s: float = float(os.getenv("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", "1.5")) + after_vehicle_open_min_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MIN_S", "0.3")) + after_vehicle_open_max_s: float = float(os.getenv("IAAI_AFTER_VEHICLE_OPEN_MAX_S", "1.0")) + between_vehicles_min_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MIN_S", "0.5")) + between_vehicles_max_s: float = float(os.getenv("IAAI_BETWEEN_VEHICLES_MAX_S", "1.5")) + after_page_change_min_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MIN_S", "3.0")) + after_page_change_max_s: float = float(os.getenv("IAAI_AFTER_PAGE_CHANGE_MAX_S", "6.0")) + + +@dataclass(slots=True) +class ListingConfig: + cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") + max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5")) + max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100")) + page_link_limit: int = int(os.getenv("IAAI_PAGE_LINK_LIMIT", "200")) + include_pagination: bool = os.getenv("IAAI_INCLUDE_PAGINATION", "true").strip().lower() in {"1", "true", "yes", "on"} + collect_current_page_only: bool = os.getenv("IAAI_COLLECT_CURRENT_PAGE_ONLY", "false").strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(slots=True) +class DatabaseConfig: + url: str = os.getenv("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper") + echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"} + pool_size: int = int(os.getenv("IAAI_DATABASE_POOL_SIZE", "5")) + max_overflow: int = int(os.getenv("IAAI_DATABASE_MAX_OVERFLOW", "10")) + + +@dataclass(slots=True) +class RedisConfig: + url: str = os.getenv("IAAI_REDIS_URL", "redis://localhost:6379/0") + + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = os.getenv("CELERY_BROKER_URL", "") or "" + result_backend: str = os.getenv("CELERY_RESULT_BACKEND", "") or "" + task_soft_time_limit: int = int(os.getenv("CELERY_TASK_SOFT_TIME_LIMIT", "600")) + task_time_limit: int = int(os.getenv("CELERY_TASK_TIME_LIMIT", "900")) + worker_concurrency: int = int(os.getenv("CELERY_WORKER_CONCURRENCY", "1")) + beat_sync_interval_minutes: int = int(os.getenv("CELERY_BEAT_SYNC_INTERVAL_MINUTES", "60")) + beat_sync_limit: int = int(os.getenv("CELERY_BEAT_SYNC_LIMIT", "26")) + + +@dataclass(slots=True) +class ProxyConfig: + server: str | None = (os.getenv("IAAI_PROXY_SERVER") or "").strip() or None + username: str | None = (os.getenv("IAAI_PROXY_USERNAME") or "").strip() or None + password: str | None = (os.getenv("IAAI_PROXY_PASSWORD") or "").strip() or None + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + +@dataclass(slots=True) +class Settings: + home_url: str = "https://www.iaai.com/" + default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000")) + network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800")) + max_retries: int = int(os.getenv("IAAI_MAX_RETRIES", "3")) + retry_delay_seconds: float = float(os.getenv("IAAI_RETRY_DELAY_SECONDS", "2.5")) + retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0")) + retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25")) + headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"} + log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO") + log_file: str | None = os.getenv("IAAI_LOG_FILE") or None + enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"} + sync_only_new: bool = os.getenv("IAAI_SYNC_ONLY_NEW", "true").strip().lower() in {"1", "true", "yes", "on"} + raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None + scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60")) + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + capture: CaptureConfig = field(default_factory=CaptureConfig) + pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) + + +settings = Settings() diff --git a/iaai_scraper/core/logs.py b/iaai_scraper/core/logs.py new file mode 100644 index 0000000..ff4ea69 --- /dev/null +++ b/iaai_scraper/core/logs.py @@ -0,0 +1,31 @@ +import logging +import sys +from contextvars import ContextVar + + +TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") + + +class TraceIdFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = TRACE_ID.get() + return True + + +def set_trace_id(trace_id: str) -> None: + TRACE_ID.set(trace_id) + + +def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)] + if log_file: + handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + trace_filter = TraceIdFilter() + for handler in handlers: + handler.addFilter(trace_filter) + logging.basicConfig( + level=getattr(logging, level.upper(), logging.INFO), + format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s", + handlers=handlers, + force=True, + ) diff --git a/iaai_scraper/core/retry.py b/iaai_scraper/core/retry.py new file mode 100644 index 0000000..1e5252d --- /dev/null +++ b/iaai_scraper/core/retry.py @@ -0,0 +1,49 @@ +import logging +import random +import time +from collections.abc import Callable +from functools import wraps +from typing import Any + +from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError + +logger = logging.getLogger("iaai_scraper.retry") + +RETRYABLE_EXCEPTIONS = ( + PlaywrightTimeoutError, + Error, + ConnectionError, + OSError, + TimeoutError, +) + + +def retryable( + max_attempts: int, + delay_seconds: float = 2.5, + backoff_multiplier: float = 2.0, + jitter_seconds: float = 0.0, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + last_error: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + return func(*args, **kwargs) + except RETRYABLE_EXCEPTIONS as exc: + last_error = exc + logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) + if attempt < max_attempts: + sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1)) + if jitter_seconds > 0: + sleep_for += random.uniform(0, jitter_seconds) + logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for) + time.sleep(sleep_for) + if last_error is not None: + raise last_error + raise RuntimeError("Retry wrapper failed without a captured exception") + + return wrapper + + return decorator diff --git a/iaai_scraper/core/utils.py b/iaai_scraper/core/utils.py new file mode 100644 index 0000000..aaf0b93 --- /dev/null +++ b/iaai_scraper/core/utils.py @@ -0,0 +1,57 @@ +import json +import random +import re +import time +from pathlib import Path +from typing import Any, Iterable + + +# Файловые утилиты +def save_to_json(data: Any, filename: str | Path) -> None: + path = Path(filename) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +# Небольшая случайная пауза между запросами +def short_sleep(a: float = 0.10, b: float = 0.35) -> None: + time.sleep(random.uniform(a, b)) + + +# Маскирование персональных данных +def mask_email(email: str) -> str: + if "@" not in email: + return "***" + local, domain = email.split("@", 1) + safe_local = local[:2] + "***" if len(local) > 2 else local[:1] + "*" + return f"{safe_local}@{domain}" + + +# Возвращает первое непустое значение +def first_non_empty(values: Iterable[Any]) -> Any | None: + for value in values: + if value not in (None, "", [], {}, ()): + return value + return None + + +# Регулярные выражения для VIN, lot и price +VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) +LOT_RE = re.compile(r"\b(\d{7,10})\b") +PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") + + +# Глубокий рекурсивный поиск значений по ключам +def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: + found = [] + if _depth >= max_depth: + return found + if isinstance(obj, dict): + for key, value in obj.items(): + if key.lower() in target_keys: + found.append(value) + found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) + elif isinstance(obj, list): + for item in obj: + found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) + return found diff --git a/iaai_scraper/parsing/__init__.py b/iaai_scraper/parsing/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/parsing/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py new file mode 100644 index 0000000..6dc02b4 --- /dev/null +++ b/iaai_scraper/parsing/mapper.py @@ -0,0 +1,412 @@ +import hashlib +import json +import re +from datetime import datetime, timezone +from typing import Any +from urllib.parse import urlparse + +from ..core.utils import first_non_empty +from ..storage.enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) +from ..storage.schemas import CarRecord, ImageRecord + + +class CarMapper: + """IAAI data → CarRecord.""" + + BODY_MAP = { + "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", + "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", + "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", + "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", + "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", + "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", + } + DRIVE_MAP = { + "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", + "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", + "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", + } + GEARBOX_MAP = { + "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", + "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", + "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", + } + STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} + COUNTRY_MAP = { + "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", + "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", + } + NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} + + def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: + # Собираем нормализованную DB-модель. + vehicle_summary = vehicle_summary or {} + payload_insights = payload_insights or {} + notes: list[str] = [] + core = payload_insights.get("vehicle_core", {}) + pricing = payload_insights.get("pricing", {}) + damage = payload_insights.get("damage", {}) + auction = payload_insights.get("auction", {}) + images = payload_insights.get("images", {}) + + origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) + parser_id = f"iaai:{origin_id}" + brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" + model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" + year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")])) + price = self._first_parsed_int( + [ + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + pricing.get("actual_cash_value"), + ], + self._to_money_int, + ) + mileage = self._first_parsed_int( + [core.get("odometer"), vehicle_summary.get("odometer"), 0], + self._to_int, + ) or 0 + color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) + drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) + gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) + steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" + body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) + engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")])) + title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) + seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) + location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""])) + country = self._normalize_country(first_non_empty([core.get("country"), location, "US"])) + is_damaged = self._bool_damage(damage, vehicle_summary) + is_sold = self._bool_sold(auction) + one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) + new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) + rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) + repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) + non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) + evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None + currency = self._normalize_currency( + first_non_empty( + [ + pricing.get("currency"), + vehicle_summary.get("currency"), + pricing.get("buy_now"), + pricing.get("current_bid"), + vehicle_summary.get("buy_now"), + vehicle_summary.get("current_bid"), + "USD", + ] + ) + ) + slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id]))) + images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) + origin = "IAAI" if "IAAI" in ORIGIN_ENUM_VALUES else "NA" + if not price: + notes.append("Price is missing or not parseable from the observed payloads.") + if not vehicle_summary.get("vin"): + notes.append("VIN was not observed in the accessible payloads for this account/session.") + if not images_records: + notes.append("No image URLs were found in the captured payloads.") + + raw_attributes = { + # Сохраняем полезный сырой контекст. + "vin": vehicle_summary.get("vin"), + "lot_number": first_non_empty([core.get("lot_number"), vehicle_summary.get("lot_number")]), + "trim": first_non_empty([core.get("trim"), vehicle_summary.get("trim")]), + "fuel_type": first_non_empty([core.get("fuel_type"), vehicle_summary.get("fuel_type")]), + "cylinders": first_non_empty([core.get("cylinders"), vehicle_summary.get("cylinders")]), + "engine": first_non_empty([core.get("engine"), vehicle_summary.get("engine")]), + "manufactured_in": vehicle_summary.get("manufactured_in"), + "vehicle_class": vehicle_summary.get("vehicle_class"), + "run_and_drive": first_non_empty([core.get("run_and_drive"), vehicle_summary.get("run_and_drive")]), + "keys": first_non_empty([core.get("keys"), vehicle_summary.get("keys")]), + "title": title_text, + "title_brand": vehicle_summary.get("title_brand"), + "damage_primary": first_non_empty([damage.get("primary"), vehicle_summary.get("primary_damage")]), + "damage_secondary": damage.get("secondary"), + "damage_description": damage.get("description"), + "buy_now": first_non_empty([pricing.get("buy_now"), vehicle_summary.get("buy_now")]), + "current_bid": first_non_empty([pricing.get("current_bid"), vehicle_summary.get("current_bid")]), + "actual_cash_value": pricing.get("actual_cash_value"), + "estimated_repair_cost": pricing.get("estimated_repair_cost"), + "seller": seller, + "location": location, + "vehicle_location": vehicle_summary.get("vehicle_location"), + "auction_date": auction.get("auction_date"), + "lane": auction.get("lane"), + "branch": auction.get("branch"), + "sale_status": auction.get("sale_status"), + "source_endpoints": payload_insights.get("source_endpoints", {}), + } + + content_hash = hashlib.sha256(json.dumps({ + # Хеш для пропуска записей без изменений. + "brand": brand, "model": model, "year": year, "price": price, "mileage": mileage, + "color": color, "drive": drive, "gearbox": gearbox, "body_type": body_type, + "engine_volume": engine_volume, "is_damaged": is_damaged, "is_sold": is_sold, + "country": country, "selling_type": "AUCTION", "one_owner": one_owner, + "new_car": new_car, "evaluation": evaluation, "non_smoking": non_smoking, + "rental": rental, "repair_history": repair_history, + "images": [image.fullres_image for image in images_records], + }, sort_keys=True, default=str).encode()).hexdigest() + + return CarRecord( + parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, + mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, + steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, + selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car, + is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, + evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, + slug=slug, last_seen_at=datetime.now(timezone.utc), content_hash=content_hash, images=images_records, + raw_attributes=raw_attributes, mapping_notes=notes, + ) + + @staticmethod + def _as_str(value: Any) -> str: + return "" if value is None else str(value).strip() + + @staticmethod + def _to_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + digits = re.sub(r"[^\d]", "", str(value)) + return int(digits) if digits else None + + @classmethod + def _to_money_int(cls, value: Any) -> int | None: + # Нормализация стоимости из разных форматов. + if value is None: + return None + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return int(value) + + text = str(value).strip() + if not text: + return None + + lowered = text.lower() + if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]): + return None + + numbers = re.findall(r"\d[\d\s.,]*", text) + if not numbers: + return None + + best: int | None = None + for number in numbers: + clean = number.replace(" ", "") + if "," in clean and "." in clean: + # Поддержка 1,234.56 и 1.234,56. + if clean.rfind(",") > clean.rfind("."): + clean = clean.replace(".", "").replace(",", ".") + else: + clean = clean.replace(",", "") + elif "," in clean: + parts = clean.split(",") + # 123,45 -> 123.45, иначе разделитель тысяч. + if len(parts[-1]) in {1, 2} and len(parts) == 2: + clean = clean.replace(",", ".") + else: + clean = clean.replace(",", "") + elif "." in clean: + parts = clean.split(".") + if not (len(parts[-1]) in {1, 2} and len(parts) == 2): + clean = clean.replace(".", "") + + try: + parsed = int(float(clean)) + except ValueError: + continue + + if parsed > 0 and (best is None or parsed > best): + best = parsed + + return best + + @staticmethod + def _first_parsed_int(values: list[Any], parser) -> int | None: + for value in values: + parsed = parser(value) + if parsed is not None: + return parsed + return None + + @staticmethod + def _to_year(value: Any) -> int | None: + parsed = CarMapper._to_int(value) + if parsed is None: + return None + if 1900 <= parsed <= 2100: + return parsed + return None + + def _to_engine_cc(self, value: Any) -> int | None: + # Поддержка литров и cc. + text = str(value).lower().strip() if value is not None else "" + if not text: + return None + if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): + return int(float(liters_match.group(1)) * 1000) + if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text): + return int(cubic_match.group(1)) + return int(text) if re.match(r"^\d+$", text) else None + + def _normalize_currency(self, value: Any) -> str: + text = self._as_str(value) + upper = text.upper() or "USD" + if any(token in text for token in ["€", "EUR"]): + return "EUR" + if any(token in text for token in ["¥", "JPY"]): + return "JPY" + if any(token in text for token in ["₩", "KRW"]): + return "KRW" + if any(token in text for token in ["£", "GBP"]): + return "GBP" + if any(token in text for token in ["₽", "RUB"]): + return "RUB" + if any(token in text for token in ["AED", "د.إ"]): + return "AED" + if any(token in text for token in ["CA$", "CAD"]): + return "CAD" + + text = upper + if text in CURRENCY_ENUM_VALUES: + return text + return "USD" if "$" in str(value) else "USD" + + def _normalize_drive(self, value: Any) -> str | None: + return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_gearbox(self, value: Any) -> str | None: + return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") + + def _normalize_steering(self, value: Any) -> str | None: + return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) + + def _normalize_body_type(self, value: Any) -> str: + return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" + + def _normalize_country(self, value: Any) -> str: + fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA" + return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback + + def _normalize_selling_type(self, value: Any) -> str: + text = self._as_str(value) or "AUCTION" + return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" + + def _map_value( + self, + value: Any, + mapping: dict[str, str], + allowed_values: tuple[str, ...], + *, + empty_default: str | None, + fallback: str | None, + ) -> str | None: + # Общий helper для enum-нормализации. + text = self._as_str(value).lower() + if not text: + return empty_default + if (mapped := mapping.get(text)) and mapped in allowed_values: + return mapped + for marker, mapped in mapping.items(): + if marker in text and mapped in allowed_values: + return mapped + return fallback + + @staticmethod + def _normalize_color(value: Any) -> str: + text = str(value).strip() if value is not None else "other" + if not text: + return "other" + if "/" in text: + text = text.split("/")[0].strip() + return text.lower() or "other" + + @staticmethod + def _boolish(value: Any) -> bool: + return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} + + def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: + for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: + text = self._as_str(value).lower() + if text and text not in self.NO_DAMAGE_MARKERS: + return True + return False + + def _bool_sold(self, auction: dict[str, Any]) -> bool: + return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) + + def _build_images(self, urls: list[Any]) -> list[ImageRecord]: + # Для imageKeys берем самый большой размер. + best_by_key: dict[str, str] = {} + key_order: list[str] = [] + non_keyed: list[str] = [] + for item in urls: + if not isinstance(item, str): + continue + url = item.strip() + if not url: + continue + if m := re.search(r'imageKeys=([^&]+)', url): + img_key = m.group(1) + w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 + existing = best_by_key.get(img_key) + if existing is None: + best_by_key[img_key] = url + key_order.append(img_key) + else: + existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 + if w > existing_w: + best_by_key[img_key] = url + elif url not in non_keyed: + non_keyed.append(url) + deduped = [best_by_key[k] for k in key_order] + non_keyed + return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] + + @staticmethod + def _make_fullres_url(url: str) -> str: + if "vis.iaai.com" in url: + return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) + return url + + @staticmethod + def _make_preview_url(url: str) -> str: + if "vis.iaai.com" in url: + preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) + if preview != url: + return preview + return url + + def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: + # Берем lot_number, затем vin, затем хвост URL. + for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]: + text = self._as_str(value) + if text: + return text + tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] + if "~" in tail: + tail = tail.split("~")[0] + return tail or self._slugify(vehicle_url) + + @staticmethod + def _slugify(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" + + diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py new file mode 100644 index 0000000..915dc5f --- /dev/null +++ b/iaai_scraper/parsing/parser.py @@ -0,0 +1,367 @@ +import html as html_module +import json +import logging +import re +from typing import Any + +from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty + +logger = logging.getLogger("iaai_scraper.parsers") + + +class VehicleParser: + """DOM + JSON парсер страницы авто.""" + + SUMMARY_KEY_MAP = { + "vin": {"vin", "vehicleidentificationnumber"}, + "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, + "year": {"year"}, + "make": {"make", "manufacturer", "brand"}, + "model": {"model"}, + "trim": {"trim", "series"}, + "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, + "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, + "secondary_damage": {"secondarydamage"}, + "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, + "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, + "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, + "actual_cash_value": {"actualcashvalue", "acv"}, + "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, + "keys": {"keys", "keystatus"}, + "title": {"titletype", "title", "documenttype"}, + "seller": {"seller", "sellername"}, + "location": {"location", "branchname", "auctionlocation", "branch"}, + "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, + "body_type": {"bodytype", "bodystyle"}, + "drive": {"driveline", "drive"}, + "gearbox": {"transmission", "gearbox"}, + "engine": {"engine", "enginevolume"}, + "fuel_type": {"fueltype", "fuel"}, + "cylinders": {"cylinders", "cylindercount"}, + "color": {"color", "primarycolor", "exteriorcolor"}, + } + + DOM_LABEL_MAP: dict[str, str] = { + "stock #": "lot_number", + "vin (status)": "vin", + "vin": "vin", + "primary damage": "primary_damage", + "secondary damage": "secondary_damage", + "odometer": "odometer", + "body style": "body_type", + "engine": "engine", + "transmission": "gearbox", + "drive line type": "drive", + "fuel type": "fuel_type", + "cylinders": "cylinders", + "exterior/interior": "color", + "exterior color": "color", + "model": "model", + "series": "trim", + "selling branch": "location", + "vehicle location": "vehicle_location", + "auction date and time": "auction_date", + "lane/run #": "lane", + "actual cash value": "actual_cash_value", + "estimated repair cost": "estimated_repair_cost", + "seller": "seller", + "title/sale doc": "title", + "title/sale doc brand": "title_brand", + "start code": "run_and_drive", + "key": "keys", + "manufactured in": "manufactured_in", + "vehicle class": "vehicle_class", + } + + def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: + result: dict[str, str] = {} + if not dom_text: + return result + lines = [line.strip() for line in dom_text.split("\n") if line.strip()] + known_labels = set(self.DOM_LABEL_MAP.keys()) + for i, line in enumerate(lines): + clean = line.rstrip(":").lower().strip() + if clean in known_labels and i + 1 < len(lines): + value = lines[i + 1].strip() + if value.rstrip(":").lower().strip() in known_labels: + continue + if value.lower() in ("more actions", "view", "print", "share", "back to results"): + continue + field_name = self.DOM_LABEL_MAP[clean] + if field_name not in result or not result[field_name]: + result[field_name] = value + return result + + def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: + result: dict[str, str | None] = {"year": None, "make": None, "model": None} + title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") + if title_match: + result["year"] = title_match.group(1) + result["make"] = title_match.group(2) + result["model"] = title_match.group(3) + return result + dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") + if dom_match: + result["year"] = dom_match.group(1) + result["make"] = dom_match.group(2) + result["model"] = dom_match.group(3).strip() + return result + + def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: + page_html = page_html or "" + dom_text = dom_text or "" + network_dump = network_dump or {} + responses = network_dump.get("json_responses", []) + payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] + dom_kv = self._parse_dom_key_value_pairs(dom_text) + page_title = "" + title_match = re.search(r"]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) + if title_match: + page_title = title_match.group(1).strip() + title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) + + summary: dict[str, Any] = {"source_url": vehicle_url} + for field, candidate_keys in self.SUMMARY_KEY_MAP.items(): + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, candidate_keys)) + if field in dom_kv: + values.append(dom_kv[field]) + summary[field] = first_non_empty(values) + + summary["year"] = summary.get("year") or title_parsed.get("year") + summary["make"] = summary.get("make") or title_parsed.get("make") + summary["model"] = summary.get("model") or title_parsed.get("model") + summary["trim"] = summary.get("trim") or dom_kv.get("trim") + summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html) + if summary.get("vin") and isinstance(summary["vin"], str): + vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip() + vin_match = VIN_RE.search(vin_clean) + summary["vin"] = vin_match.group(1) if vin_match else vin_clean + summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) + summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) + for dom_field, dom_value in dom_kv.items(): + if dom_field not in summary or not summary[dom_field]: + summary[dom_field] = dom_value + prices = self._extract_prices_from_text(dom_text) + if not summary.get("actual_cash_value") and prices: + summary["actual_cash_value"] = prices[0] + if not summary.get("buy_now"): + buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE) + if buy_now_match: + summary["buy_now"] = buy_now_match.group(1) + elif prices: + summary["buy_now"] = prices[0] + if not summary.get("current_bid") and len(prices) > 1: + summary["current_bid"] = prices[1] + + embedded = self._extract_embedded_json(page_html) + for item in embedded: + p = item.get("payload") + if isinstance(p, (dict, list)): + payloads.append(p) + for field, candidate_keys in self.SUMMARY_KEY_MAP.items(): + if not summary.get(field): + val = first_non_empty(deep_find_key(p, candidate_keys)) + if val: + summary[field] = val + + return { + "vehicle_summary": summary, + "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url), + "embedded_json": embedded, + "dom_hints": self._dom_hints(dom_text), + "access_notes": self._build_access_notes(summary, responses), + } + + def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]: + image_urls = self._extract_image_urls(payloads, "", vehicle_url) + return { + "vehicle_core": { + "vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"), + "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), + "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), + "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), + "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), + "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), + "color": summary.get("color"), "keys": summary.get("keys"), + }, + "pricing": { + "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), + "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), + "currency": self._guess_currency(summary), + }, + "bids": self._build_bid_insights(summary, payloads), + "damage": { + "primary": summary.get("primary_damage"), + "secondary": summary.get("secondary_damage"), + "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), + }, + "auction": { + "auction_date": summary.get("auction_date"), + "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), + "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), + "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), + "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), + }, + "images": {"count": len(image_urls), "urls": image_urls}, + "source_endpoints": self._build_source_endpoints(responses), + } + + def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: + return { + "amount": summary.get("current_bid"), + "currency": self._guess_currency(summary), + "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), + "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), + } + + def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: + mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} + for item in responses: + url = item.get("url", "") + category = item.get("category", "other") + if category == "vehicle": + mapping["vehicle"].append(url) + lowered = url.lower() + if any(token in lowered for token in ["bid", "offer"]): + mapping["bids"].append(url) + if any(token in lowered for token in ["damage", "report"]): + mapping["damage"].append(url) + if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): + mapping["auction"].append(url) + elif category in mapping: + mapping[category].append(url) + return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} + + def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: + endpoints = [item.get("url", "") for item in responses] + return { + "vin_visible": bool(summary.get("vin")), + "images_visible": bool(summary.get("image_urls")), + "network_json_count": len(responses), + "possible_captcha": False, + "possible_antibot": False, + "observed_endpoints": endpoints[:20], + } + + @staticmethod + def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: + lowered = {key.lower() for key in keys} + values: list[Any] = [] + for payload in payloads: + values.extend(deep_find_key(payload, lowered)) + return values + + @staticmethod + def _guess_currency(summary: dict[str, Any]) -> str: + for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: + value = str(summary.get(key) or "") + if "$" in value: + return "USD" + if "€" in value: + return "EUR" + if "¥" in value: + return "JPY" + return "USD" + + @staticmethod + def _extract_vin(text: str) -> str | None: + match = VIN_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_lot_number(text: str) -> str | None: + match = LOT_RE.search(text or "") + return match.group(1) if match else None + + @staticmethod + def _extract_prices_from_text(text: str) -> list[str]: + return [match.group(1) for match in PRICE_RE.finditer(text or "")] + + @staticmethod + def _extract_embedded_json(html: str) -> list[dict[str, Any]]: + scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) + extracted: list[dict[str, Any]] = [] + for script_text in scripts: + if "{" not in script_text and "[" not in script_text: + continue + try: + parsed = json.loads(script_text.strip()) + except Exception: + continue + extracted.append({"type": "inline_json", "payload": parsed}) + return extracted + + @staticmethod + def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: + vehicle_key = "" + key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") + if key_match: + vehicle_key = key_match.group(1) + found: list[str] = [] + for payload in payloads: + found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) + flat: list[str] = [] + seen_flat: set[str] = set() + for item in found: + if isinstance(item, str) and item.startswith("http"): + cleaned = html_module.unescape(item) + lowered = cleaned.lower() + if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + elif isinstance(item, list): + for child in item: + if isinstance(child, str) and child.startswith("http"): + cleaned = html_module.unescape(child) + lowered = cleaned.lower() + if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + continue + if cleaned not in seen_flat: + seen_flat.add(cleaned) + flat.append(cleaned) + for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: + for match in re.finditer(pattern, html or "", re.IGNORECASE): + url = html_module.unescape(match.group(1).strip()) + if not url.startswith("http") or url in seen_flat: + continue + lowered = url.lower() + if vehicle_key and vehicle_key in url: + seen_flat.add(url) + flat.append(url) + elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]): + if vehicle_key and vehicle_key not in url: + continue + seen_flat.add(url) + flat.append(url) + if vehicle_key: + for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""): + cleaned = html_module.unescape(url) + if cleaned not in seen_flat and vehicle_key in cleaned: + seen_flat.add(cleaned) + flat.append(cleaned) + filtered: list[str] = [] + for url in flat: + lowered = url.lower() + if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): + continue + if "vis.iaai.com" in lowered and "/resizer" not in lowered: + continue + filtered.append(url) + return filtered + + @staticmethod + def _dom_hints(text: str) -> dict[str, Any]: + lowered = (text or "").lower() + return { + "has_buy_now_text": "buy now" in lowered, + "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, + "has_damage_text": "damage" in lowered, + "has_vin_text": "vin" in lowered, + "has_title_text": "title" in lowered, + "has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]), + "has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]), + } diff --git a/iaai_scraper/proxy_bridge.py b/iaai_scraper/proxy_bridge.py new file mode 100644 index 0000000..e2b0b99 --- /dev/null +++ b/iaai_scraper/proxy_bridge.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import logging +import os +import select +import socket +import socketserver +import struct +import threading +from urllib.parse import urlsplit + +BUFFER_SIZE = 65536 +CRLF = b"\r\n" +DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "0.0.0.0") +DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899")) +SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "") +SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002")) +SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "") +SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "") +RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60")) +MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64")) + +logger = logging.getLogger("proxy_bridge") + + +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + allow_reuse_address = True + daemon_threads = True + + def __init__(self, server_address, request_handler_class): + super().__init__(server_address, request_handler_class) + self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS) + + def process_request_thread(self, request, client_address): + with self._worker_semaphore: + super().process_request_thread(request, client_address) + + +def _recv_exact(sock: socket.socket, size: int) -> bytes: + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise ConnectionError("Unexpected EOF from SOCKS5 server") + data += chunk + return data + + +def _socks5_connect(host: str, port: int) -> socket.socket: + if not SOCKS5_HOST: + raise RuntimeError("SOCKS5_PROXY_HOST is not configured") + + upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30) + upstream.settimeout(30) + + methods = [0x00] + if SOCKS5_USER or SOCKS5_PASS: + methods = [0x02] + upstream.sendall(bytes([0x05, len(methods), *methods])) + version, method = _recv_exact(upstream, 2) + if version != 0x05 or method == 0xFF: + upstream.close() + raise ConnectionError("SOCKS5 authentication negotiation failed") + + if method == 0x02: + username = SOCKS5_USER.encode("utf-8") + password = SOCKS5_PASS.encode("utf-8") + if len(username) > 255 or len(password) > 255: + upstream.close() + raise ValueError("SOCKS5 username/password too long") + upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password) + auth_version, auth_status = _recv_exact(upstream, 2) + if auth_version != 0x01 or auth_status != 0x00: + upstream.close() + raise ConnectionError("SOCKS5 username/password authentication failed") + + try: + socket.inet_aton(host) + addr_type = 0x01 + addr_payload = socket.inet_aton(host) + except OSError: + host_bytes = host.encode("idna") + if len(host_bytes) > 255: + upstream.close() + raise ValueError("Target host is too long for SOCKS5 domain format") + addr_type = 0x03 + addr_payload = bytes([len(host_bytes)]) + host_bytes + + request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port) + upstream.sendall(request) + + response_head = _recv_exact(upstream, 4) + version, reply, _reserved, reply_addr_type = response_head + if version != 0x05 or reply != 0x00: + upstream.close() + raise ConnectionError(f"SOCKS5 connect failed with code {reply}") + + if reply_addr_type == 0x01: + _recv_exact(upstream, 4) + elif reply_addr_type == 0x03: + domain_len = _recv_exact(upstream, 1)[0] + _recv_exact(upstream, domain_len) + elif reply_addr_type == 0x04: + _recv_exact(upstream, 16) + _recv_exact(upstream, 2) + + upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + return upstream + + +def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None: + sockets = [left, right] + left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) + try: + while True: + readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS) + if exceptional: + break + if not readable: + logger.debug("Relay idle timeout reached; closing sockets") + return + for current in readable: + other = right if current is left else left + data = current.recv(BUFFER_SIZE) + if not data: + return + other.sendall(data) + finally: + for sock in sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + try: + sock.close() + except OSError: + pass + + +class ProxyHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + try: + request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip() + if not request_line: + return + + method, target, version = request_line.split() + headers = self._read_headers() + logger.info("%s %s", method, target) + + if method.upper() == "CONNECT": + host, port = self._parse_connect_target(target) + logger.info("CONNECT %s:%s", host, port) + upstream = _socks5_connect(host, port) + self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF) + self.wfile.flush() + _relay_bidirectional(self.connection, upstream) + return + + host, port, path = self._parse_forward_target(target, headers) + upstream = _socks5_connect(host, port) + self._send_forward_request(upstream, method, path, version, headers) + body = self._read_request_body(headers) + if body: + upstream.sendall(body) + _relay_bidirectional(self.connection, upstream) + except Exception as exc: + logger.exception("Proxy bridge request failed: %s", exc) + try: + self.wfile.write( + b"HTTP/1.1 502 Bad Gateway" + CRLF + + b"Connection: close" + CRLF + + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF + + b"Bad Gateway" + ) + self.wfile.flush() + except OSError: + pass + + def _read_headers(self) -> list[tuple[str, str]]: + headers: list[tuple[str, str]] = [] + while True: + line = self.rfile.readline(BUFFER_SIZE) + if line in {CRLF, b"\n", b""}: + break + decoded = line.decode("iso-8859-1") + if ":" not in decoded: + continue + name, value = decoded.split(":", 1) + headers.append((name.strip(), value.strip())) + return headers + + @staticmethod + def _parse_connect_target(target: str) -> tuple[str, int]: + if target.startswith("["): + end = target.find("]") + if end == -1 or len(target) <= end + 2 or target[end + 1] != ":": + raise ValueError("Invalid CONNECT target") + host = target[1:end] + port_text = target[end + 2 :] + return host, int(port_text) + + host, port_text = target.rsplit(":", 1) + return host, int(port_text) + + @staticmethod + def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]: + if target.startswith("http://"): + parts = urlsplit(target) + port = parts.port or 80 + path = parts.path or "/" + if parts.query: + path += f"?{parts.query}" + return parts.hostname or "", port, path + + if target.startswith("https://"): + raise ValueError("HTTPS absolute-form request must use CONNECT") + + host_header = next((value for name, value in headers if name.lower() == "host"), "") + if not host_header: + raise ValueError("Missing Host header") + if ":" in host_header: + host, port_text = host_header.rsplit(":", 1) + return host, int(port_text), target + return host_header, 80, target + + def _send_forward_request( + self, + upstream: socket.socket, + method: str, + path: str, + version: str, + headers: list[tuple[str, str]], + ) -> None: + filtered_headers: list[tuple[str, str]] = [] + hop_by_hop = { + "proxy-connection", + "proxy-authorization", + "connection", + "keep-alive", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } + for name, value in headers: + if name.lower() in hop_by_hop: + continue + filtered_headers.append((name, value)) + + request_head = [f"{method} {path} {version}\r\n"] + request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers) + request_head.append("\r\n") + upstream.sendall("".join(request_head).encode("iso-8859-1")) + + def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes: + transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "") + if "chunked" in transfer_encoding.lower(): + return self._read_chunked_request_body() + + content_length = next((value for name, value in headers if name.lower() == "content-length"), None) + if not content_length: + return b"" + return self.rfile.read(int(content_length)) + + def _read_chunked_request_body(self) -> bytes: + chunks: list[bytes] = [] + while True: + size_line = self.rfile.readline(BUFFER_SIZE) + if not size_line: + raise ConnectionError("Unexpected EOF in chunked request") + size_text = size_line.strip().split(b";", 1)[0] + chunk_size = int(size_text, 16) + chunks.append(size_line) + if chunk_size == 0: + while True: + trailer_line = self.rfile.readline(BUFFER_SIZE) + if not trailer_line: + raise ConnectionError("Unexpected EOF in chunked trailers") + chunks.append(trailer_line) + if trailer_line in {CRLF, b"\n"}: + return b"".join(chunks) + + chunk_data = self.rfile.read(chunk_size) + if len(chunk_data) != chunk_size: + raise ConnectionError("Unexpected EOF in chunk body") + chunks.append(chunk_data) + chunk_end = self.rfile.read(2) + if chunk_end != CRLF: + raise ConnectionError("Invalid chunk terminator") + chunks.append(chunk_end) + + +def main() -> None: + if not SOCKS5_HOST: + raise SystemExit("SOCKS5_PROXY_HOST is required") + + logging.basicConfig( + level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(), + format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s", + ) + + with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server: + logger.info( + "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)", + DEFAULT_LISTEN_HOST, + DEFAULT_LISTEN_PORT, + SOCKS5_HOST, + SOCKS5_PORT, + MAX_WORKERS, + ) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py new file mode 100644 index 0000000..5b14f51 --- /dev/null +++ b/iaai_scraper/scraper.py @@ -0,0 +1,460 @@ +import logging +import re +import signal +import time +import uuid + +from playwright.sync_api import Error as PlaywrightError +from playwright.sync_api import BrowserContext, Page, sync_playwright +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + +from .browser import BrowserFactory, HumanPacer, NetworkCapture +from .core.config import Settings, settings +from .core.logs import set_trace_id, setup_logging +from .core.retry import retryable +from .core.utils import save_to_json +from .parsing.mapper import CarMapper +from .parsing.parser import VehicleParser +from .storage.db import PersistenceService +from .browser.listing import ListingCollector +from .storage.schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.scraper") +VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) + + +class IAAIScraper: + + @staticmethod + def _extract_origin_id_from_url(vehicle_url: str) -> str | None: + match = VEHICLE_ID_RE.search(vehicle_url) + if not match: + return None + return match.group(1) + + def __init__(self, runtime_settings: Settings | None = None) -> None: + # Базовые зависимости и сервисы скрапера + self.settings = runtime_settings or settings + setup_logging(self.settings.log_level, self.settings.log_file) + self.trace_id = uuid.uuid4().hex[:12] + set_trace_id(self.trace_id) + self.playwright = None + self.browser = None + self.context: BrowserContext | None = None + self.browser_factory = BrowserFactory(self.settings) + self.pacer = HumanPacer(self.settings) + self.listing_collector = ListingCollector(self.settings, self.pacer) + self.vehicle_parser = VehicleParser() + self.car_mapper = CarMapper() + self.persistence = PersistenceService(self.settings) + self._shutdown_requested = False + + # Жизненный цикл браузера + + def _new_trace_id(self, prefix: str) -> str: + # Новый trace id для отдельной операции + trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" + self.trace_id = trace_id + set_trace_id(trace_id) + return trace_id + + def __enter__(self) -> "IAAIScraper": + if self.playwright is None: + self.playwright = sync_playwright().start() + if self.browser is None: + self.browser = self.browser_factory.create_browser(self.playwright) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + def close(self) -> None: + # Закрываем ресурсы аккуратно даже при ошибках Playwright + if self.context is not None: + try: + self.context.close() + except PlaywrightError: + pass + finally: + self.context = None + if self.browser is not None: + try: + self.browser.close() + except PlaywrightError: + pass + finally: + self.browser = None + if self.playwright is not None: + try: + self.playwright.stop() + except PlaywrightError: + pass + finally: + self.playwright = None + + def _new_context(self, storage_state: str | None = None) -> BrowserContext: + # Контекст пересоздаётся, чтобы не тянуть старое состояние страницы + if self.browser is None: + self.__enter__() + if self.context: + self.context.close() + self.context = self.browser_factory.create_context(self.browser, storage_state=storage_state) + return self.context + + def init_db(self): + self.persistence.create_tables() + return {"status": "ok", "database_url": self.settings.database.url} + + def _get_unauthenticated_page(self) -> Page: + context = self._new_context() + return context.new_page() + + # Сбор листинга + + def collect_listing(self, make: str | None = None, model: str | None = None): + # Собираем ссылки карточек с публичного листинга + page = self._get_unauthenticated_page() + + try: + listing = self.listing_collector.collect_listing_links(page, make=make, model=model) + finally: + page.close() + + return { + "status": "ok", + **listing, + } + + def _get_page(self) -> Page: + if not self.context: + self._new_context() + return self.context.new_page() + + # Открытие и скрейп карточки + + @retryable(max_attempts=3, jitter_seconds=0.25) + def open_vehicle_page(self, vehicle_url: str): + # Лёгкое открытие страницы без сетевого дампа и сохранения в БД + trace_id = self._new_trace_id("open") + started_at = time.perf_counter() + page = self._get_page() + try: + self.pacer.before_vehicle_open() + page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=15000) + except PlaywrightTimeoutError: + pass + self.pacer.after_vehicle_open() + + html = page.content() + try: + dom_text = page.locator("body").inner_text(timeout=10_000) + except Exception: + dom_text = "" + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, {"json_responses": []}) + return { + "trace_id": trace_id, + "source_url": vehicle_url, + "opened_in_light_mode": True, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + **parsed, + } + finally: + page.close() + + @retryable(max_attempts=3, jitter_seconds=0.25) + def scrape_vehicle_detail(self, vehicle_url: str): + """Открыть страницу, перехватить JSON, вернуть данные.""" + page = self._get_page() + try: + return self._scrape_on_page(page, vehicle_url) + finally: + page.close() + + def _scrape_on_page(self, page: Page, vehicle_url: str): + # Полный проход по карточке с network capture и маппингом в DB-модель + trace_id = self._new_trace_id("scrape") + started_at = time.perf_counter() + capture = NetworkCapture(self.settings) + capture.attach(page, origin_url=vehicle_url) + + page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=10000) + except PlaywrightTimeoutError: + pass + time.sleep(self.settings.network_settle_ms / 1000) + + html = page.content() + try: + dom_text = page.locator("body").inner_text(timeout=10_000) + except Exception: + dom_text = "" + network_dump = capture.export() + parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + **parsed, + "db_record": db_record.model_dump(mode="json"), + } + + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + # Синхронизация с БД + + def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"): + """Scrape + upsert одного авто.""" + # Один URL -> один scrape -> один upsert + trace_id = self._new_trace_id("sync-vehicle") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + ids_fetched = 1 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + status = "failed" + error_summary = None + try: + scrape_result = self.scrape_vehicle_detail(vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + cars_upserted = 1 + images_upserted = int(upsert.get("images_upserted", 0)) + status = "success" + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + "vehicle_url": vehicle_url, + "db_action": upsert.get("action"), + "images_upserted": images_upserted, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "db_record": db_record, + } + except Exception as exc: + cars_failed = 1 + status = "failed" + error_summary = str(exc) + raise + finally: + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=ids_fetched, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + def sync_listing( + self, + make: str | None = None, + model: str | None = None, + lane: str = "iaai_cars", + limit: int | None = None, + only_new: bool | None = None, + ): + """Листинг + sync всех найденных машин.""" + # Массовая синхронизация с общим run_id и сбором ошибок + trace_id = self._new_trace_id("sync-listing") + started_at = time.perf_counter() + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane=lane) + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + total = 0 + skipped_existing = 0 + failures: list[dict[str, str]] = [] + listing: dict = {} + + try: + listing = self.collect_listing(make=make, model=model) + vehicle_urls = list(listing.get("vehicle_urls", [])) + if limit is not None: + vehicle_urls = vehicle_urls[:max(0, limit)] + + effective_only_new = self.settings.sync_only_new if only_new is None else only_new + if effective_only_new: + existing_urls = self.persistence.get_existing_origin_urls(vehicle_urls) + url_to_origin_id = { + url: self._extract_origin_id_from_url(url) + for url in vehicle_urls + } + candidate_origin_ids = [origin_id for origin_id in url_to_origin_id.values() if origin_id] + existing_ids = self.persistence.get_existing_origin_ids(candidate_origin_ids) + + known_urls = { + url + for url in vehicle_urls + if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) + } + + skipped_existing = len(known_urls) + if skipped_existing: + logger.info("Filtering already known vehicles: skipped %d", skipped_existing) + vehicle_urls = [url for url in vehicle_urls if url not in known_urls] + + total = len(vehicle_urls) + logger.info("Starting sync: %d vehicles to process", total) + + for index, vehicle_url in enumerate(vehicle_urls, start=1): + page = self._get_page() + try: + logger.info("[%d/%d] Scraping %s", index, total, vehicle_url) + scrape_result = self._scrape_on_page(page, vehicle_url) + db_record = scrape_result.get("db_record") + if not db_record: + raise RuntimeError("Scrape result does not contain db_record") + record = CarRecord.model_validate(db_record) + upsert = self.persistence.upsert_car(record) + if upsert.get("action") != "skipped": + cars_upserted += 1 + img_count = int(upsert.get("images_upserted", 0)) + images_upserted += img_count + logger.info( + "[%d/%d] %s %s: %s %s %s — %s, %d images", + index, total, upsert.get("action", "?"), + record.origin_id, record.brand, record.model, + record.year or "?", record.price or "N/A", img_count, + ) + except Exception as exc: + cars_failed += 1 + failures.append({"vehicle_url": vehicle_url, "error": str(exc)}) + logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc) + finally: + try: + page.close() + except Exception: + pass + + if index < total: + self.pacer.between_vehicles() + except Exception as exc: + # Если падает collect_listing, считаем это общей ошибкой запуска + if not failures: + failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) + logger.error("sync_listing failed: %s", exc) + finally: + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + error_summary=error_summary, + ) + + logger.info( + "Sync run #%d finished: %d/%d upserted, %d failed, %d images", + run_id, cars_upserted, total, cars_failed, images_upserted, + ) + return { + "trace_id": trace_id, + "status": status, + "run_id": run_id, + "listing": listing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": skipped_existing, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + + # Встроенный планировщик + + def run_scheduled(self) -> None: + # Простой бесконечный цикл без внешнего планировщика + # Graceful shutdown по SIGINT/SIGTERM + def _handle_shutdown(signum, frame): + logger.info("Received signal %s, shutting down gracefully...", signum) + self._shutdown_requested = True + + signal.signal(signal.SIGINT, _handle_shutdown) + signal.signal(signal.SIGTERM, _handle_shutdown) + + interval = self.settings.scheduler_interval_minutes * 60 + logger.info( + "Scheduler started: syncing every %d minutes", + self.settings.scheduler_interval_minutes, + ) + cycle = 0 + while not self._shutdown_requested: + cycle += 1 + logger.info("=== Scheduler cycle #%d starting ===", cycle) + start = time.time() + try: + # Сбрасываем контекст перед каждым циклом + if self.context: + try: + self.context.close() + except PlaywrightError: + pass + self.context = None + + # Проверяем, что browser/playwright живы, и пересоздаём при необходимости + if self.browser is None or self.playwright is None: + logger.info("Browser/Playwright not available, re-initializing...") + self.close() + self.__enter__() + + result = self.sync_listing() + elapsed = time.time() - start + logger.info( + "Cycle #%d done in %.1fs: %d upserted, %d failed", + cycle, elapsed, + result.get("cars_upserted", 0), + result.get("cars_failed", 0), + ) + except Exception as exc: + elapsed = time.time() - start + logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) + # Полный сброс при любой ошибке цикла, следующий цикл пересоздаст всё + try: + self.close() + except Exception: + pass + # Пересоздаём browser для следующего цикла + try: + self.__enter__() + except Exception as reinit_exc: + logger.error("Failed to re-initialize browser: %s", reinit_exc) + + if self._shutdown_requested: + break + + sleep_time = max(0, interval - (time.time() - start)) + if sleep_time > 0: + logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) + # Прерываемый sleep, проверяем shutdown каждые 5 секунд + slept = 0.0 + while slept < sleep_time and not self._shutdown_requested: + chunk = min(5.0, sleep_time - slept) + time.sleep(chunk) + slept += chunk + + logger.info("Scheduler stopped gracefully after %d cycles.", cycle) diff --git a/iaai_scraper/storage/__init__.py b/iaai_scraper/storage/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/iaai_scraper/storage/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py new file mode 100644 index 0000000..e86640a --- /dev/null +++ b/iaai_scraper/storage/db.py @@ -0,0 +1,216 @@ +import json +import logging +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Iterator + +from sqlalchemy import create_engine, or_, select +from sqlalchemy.orm import Session, sessionmaker + +from ..core.config import Settings +from .models import Base, Car, Image, ScrapeTask, SyncRun +from .schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.db") + + +# Поля Car, которые приходят из CarRecord (без id, images, relationship). +CAR_DB_FIELDS = { + col.key for col in Car.__table__.columns + if col.key not in ("id",) +} + + +class PersistenceService: + + def __init__(self, settings: Settings) -> None: + # Инициализация engine и фабрики сессий. + self.settings = settings + engine_kwargs = { + "echo": settings.database.echo, + "future": True, + } + # Пул только для PostgreSQL. + if "postgresql" in settings.database.url: + engine_kwargs["pool_size"] = settings.database.pool_size + engine_kwargs["max_overflow"] = settings.database.max_overflow + self.engine = create_engine(settings.database.url, **engine_kwargs) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) + + def create_tables(self) -> None: + try: + Base.metadata.create_all(self.engine) + except Exception: + # Alembic уже создал таблицы/ENUM — пропускаем. + logger.debug("create_tables skipped (schema already exists)") + + @contextmanager + def session_scope(self) -> Iterator[Session]: + # Единая точка commit/rollback для операций записи. + session = self.session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def start_sync_run(self, lane: str) -> int: + # Создаём запись о запуске синхронизации. + with self.session_scope() as session: + # Если предыдущий процесс умер, оставив run в `running`, + # помечаем его как failed перед новым запуском. + now = datetime.now(timezone.utc) + stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all() + for stale in stale_runs: + stale.status = "failed" + stale.finished_at = now + if not stale.error_summary: + stale.error_summary = "Recovered stale running sync run before starting a new run" + + run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) + session.add(run) + session.flush() + return int(run.id) + + def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + # Завершаем sync_run и фиксируем итоговую статистику. + with self.session_scope() as session: + run = session.get(SyncRun, run_id) + if run is None: + return + run.finished_at = datetime.now(timezone.utc) + run.status = status + run.ids_fetched = ids_fetched + run.cars_upserted = cars_upserted + run.cars_failed = cars_failed + run.images_upserted = images_upserted + run.error_summary = error_summary + + def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]: + # Возвращает уже существующие в БД origin_url для фильтрации only-new запусков. + if not origin_urls: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: + # Возвращает уже существующие в БД origin_id для фильтрации только новых авто. + if not origin_ids: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all() + return {str(row[0]) for row in rows if row and row[0]} + + @staticmethod + def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: + for image_payload in images: + session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id)) + + @staticmethod + def _car_payload(record: CarRecord) -> dict[str, object]: + payload = record.model_dump(mode="python") + result = {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + + if "raw_attributes" in result and isinstance(result["raw_attributes"], dict): + result["raw_attributes"] = json.dumps(result["raw_attributes"], ensure_ascii=False, default=str) + return result + + def upsert_car(self, record: CarRecord): + """Insert/update/skip по content_hash.""" + # В БД отправляем только поля, реально существующие в финальной схеме cars. + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + content_hash = str(payload.get("content_hash") or "") + with self.session_scope() as session: + # Сначала пытаемся найти по origin_id, а если ранее origin_id был неполный, + # подхватываем существующую запись по origin_url, чтобы не плодить дубли. + car = session.execute( + select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) + ).scalar_one_or_none() + action = "inserted" + if car is None: + car = Car(**payload) + session.add(car) + session.flush() + else: + # Если контент не менялся, просто обновляем last_seen_at. + if content_hash and car.content_hash == content_hash: + car.last_seen_at = record.last_seen_at + session.flush() + return {"car_id": int(car.id), "images_upserted": 0, "action": "skipped"} + action = "updated" + # Обновляем поля машины и затем безопасно пересобираем картинки. + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + session.flush() + # замена картинок в savepoint + nested = session.begin_nested() + try: + for image in list(car.images): + session.delete(image) + session.flush() + self._add_images(session, int(car.id), images) + session.flush() + nested.commit() + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id) + images = [] + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + self._add_images(session, int(car.id), images) + session.flush() + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + + # --- ScrapeTask. + + def create_scrape_task(self, celery_task_id: str, task_type: str, vehicle_url: str | None = None) -> int: + """Создаёт запись задачи Celery.""" + with self.session_scope() as session: + task = ScrapeTask( + celery_task_id=celery_task_id, + task_type=task_type, + vehicle_url=vehicle_url, + status="pending", + ) + session.add(task) + session.flush() + return int(task.id) + + def update_scrape_task(self, celery_task_id: str, **kwargs) -> None: + """Обновляет поля задачи по celery_task_id.""" + with self.session_scope() as session: + task = session.execute( + select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id) + ).scalars().first() + if task is None: + return + for key, value in kwargs.items(): + if hasattr(task, key): + setattr(task, key, value) + session.flush() + + def get_scrape_task(self, celery_task_id: str) -> dict | None: + """Возвращает информацию о задаче.""" + with self.session_scope() as session: + task = session.execute( + select(ScrapeTask).where(ScrapeTask.celery_task_id == celery_task_id) + ).scalars().first() + if task is None: + return None + return { + "id": task.id, + "celery_task_id": task.celery_task_id, + "task_type": task.task_type, + "status": task.status, + "vehicle_url": task.vehicle_url, + "created_at": task.created_at.isoformat() if task.created_at else None, + "started_at": task.started_at.isoformat() if task.started_at else None, + "finished_at": task.finished_at.isoformat() if task.finished_at else None, + "result_summary": task.result_summary, + "error_message": task.error_message, + } diff --git a/iaai_scraper/storage/enums.py b/iaai_scraper/storage/enums.py new file mode 100644 index 0000000..e635e00 --- /dev/null +++ b/iaai_scraper/storage/enums.py @@ -0,0 +1,8 @@ +CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") +DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA") +GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") +STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA") +BODY_TYPE_ENUM_VALUES = ("COUPE", "SUV", "HATCHBACK", "MINIVAN", "SEDAN", "STATION_WAGON", "PICKUP", "TRUCK", "OPEN", "RV", "OTHER", "NA") +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") +ORIGIN_ENUM_VALUES = ("TAU", "CARSENSOR", "HANAMARU", "ENCAR", "KURUMA_TRADER", "ASNET", "KABABA", "ACV", "COPART", "IAAI", "NA") +SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA") diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py new file mode 100644 index 0000000..4894301 --- /dev/null +++ b/iaai_scraper/storage/models.py @@ -0,0 +1,99 @@ +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from .enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) + + +class Base(DeclarativeBase): + # База ORM. + pass + + +class Car(Base): + # Автомобиль. + __tablename__ = "cars" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + brand: Mapped[str] = mapped_column(String(50), nullable=False) + model: Mapped[str] = mapped_column(String(50), nullable=False) + year: Mapped[int | None] = mapped_column(Integer, nullable=True) + price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD") + mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + color: Mapped[str] = mapped_column(String(), nullable=False, default="other") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER") + engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) + is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) + non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + slug: Mapped[str] = mapped_column(String(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="", index=True) + raw_attributes: Mapped[str | None] = mapped_column(Text, nullable=True) + images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") + + +class Image(Base): + # Картинки автомобиля. + __tablename__ = "images" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + fullres_image: Mapped[str] = mapped_column(String(), nullable=False) + preview_image: Mapped[str] = mapped_column(String(), nullable=False) + order_index: Mapped[int] = mapped_column(Integer, nullable=False) + car_id: Mapped[int] = mapped_column(BigInteger, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False) + car: Mapped[Car] = relationship("Car", back_populates="images") + + +class SyncRun(Base): + # Статистика синхронизаций. + __tablename__ = "sync_runs" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False) + lane: Mapped[str] = mapped_column(Text, nullable=False) + ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) + + +class ScrapeTask(Base): + # Задачи Celery. + __tablename__ = "scrape_tasks" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + celery_task_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True, index=True) + task_type: Mapped[str] = mapped_column(String(50), nullable=False) # sync_vehicle/sync_listing + status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending") # pending/running/success/failed + vehicle_url: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + result_summary: Mapped[str | None] = mapped_column(Text, nullable=True) + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/iaai_scraper/storage/schemas.py b/iaai_scraper/storage/schemas.py new file mode 100644 index 0000000..43cc1b9 --- /dev/null +++ b/iaai_scraper/storage/schemas.py @@ -0,0 +1,78 @@ +from datetime import datetime, timezone +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +class ImageRecord(BaseModel): + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRecord(BaseModel): + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str + origin_id: str + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str + last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + content_hash: str = "" + images: list[ImageRecord] = Field(default_factory=list) + raw_attributes: dict[str, Any] = Field(default_factory=dict) + mapping_notes: list[str] = Field(default_factory=list) + + +class CarRead(BaseModel): + # Ответ API по авто. + model_config = ConfigDict(from_attributes=True) + + id: int + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + origin: str = "NA" + origin_url: str = "" + origin_id: str = "" + is_damaged: bool = False + slug: str = "" + last_seen_at: datetime | None = None + created_at: datetime | None = None + updated_at: datetime | None = None + diff --git a/iaai_scraper/worker/__init__.py b/iaai_scraper/worker/__init__.py new file mode 100644 index 0000000..841be13 --- /dev/null +++ b/iaai_scraper/worker/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/iaai_scraper/worker/celery_app.py b/iaai_scraper/worker/celery_app.py new file mode 100644 index 0000000..7693174 --- /dev/null +++ b/iaai_scraper/worker/celery_app.py @@ -0,0 +1,51 @@ +from celery import Celery + +from ..core.config import settings + + +def _broker_url() -> str: + return settings.celery.broker_url or settings.redis.url + + +def _result_backend() -> str: + return settings.celery.result_backend or settings.redis.url + + +celery_app = Celery( + "iaai_scraper", + broker=_broker_url(), + backend=_result_backend(), +) + +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + + task_soft_time_limit=settings.celery.task_soft_time_limit, + task_time_limit=settings.celery.task_time_limit, + worker_concurrency=settings.celery.worker_concurrency, + + # prefork + 1 процесс, тк playwright sync api + worker_pool="prefork", + worker_prefetch_multiplier=1, + + broker_connection_retry_on_startup=True, + + beat_schedule={ + "periodic-sync-listing": { + "task": "iaai_scraper.worker.tasks.sync_listing_task", + "schedule": settings.celery.beat_sync_interval_minutes * 60.0, + "kwargs": {"limit": settings.celery.beat_sync_limit}, + "options": {"queue": "scraping"}, + }, + }, + + task_routes={ + "iaai_scraper.worker.tasks.*": {"queue": "scraping"}, + }, +) + +celery_app.autodiscover_tasks(["iaai_scraper.worker"]) diff --git a/iaai_scraper/worker/tasks.py b/iaai_scraper/worker/tasks.py new file mode 100644 index 0000000..220a468 --- /dev/null +++ b/iaai_scraper/worker/tasks.py @@ -0,0 +1,133 @@ +import json +import logging +from datetime import datetime, timezone + +from celery import shared_task + +from ..core.config import Settings +from ..scraper import IAAIScraper +from ..storage.db import PersistenceService + +logger = logging.getLogger("iaai_scraper.worker.tasks") + + +def _get_persistence() -> PersistenceService: + return PersistenceService(Settings()) + + +@shared_task( + name="iaai_scraper.worker.tasks.sync_vehicle_task", + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"): + task_id = self.request.id + persistence = _get_persistence() + persistence.create_tables() + + # регистрируем запуск + persistence.update_scrape_task( + task_id, + status="running", + started_at=datetime.now(timezone.utc), + ) + + try: + with IAAIScraper() as scraper: + result = scraper.sync_vehicle(vehicle_url, lane=lane) + + persistence.update_scrape_task( + task_id, + status="success", + finished_at=datetime.now(timezone.utc), + result_summary=json.dumps({ + "car_upserted": True, + "db_action": result.get("db_action"), + "images_upserted": result.get("images_upserted", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + }, default=str), + ) + logger.info("sync_vehicle_task completed: %s", vehicle_url) + return { + "status": "success", + "vehicle_url": vehicle_url, + "db_action": result.get("db_action"), + "images_upserted": result.get("images_upserted", 0), + } + + except Exception as exc: + persistence.update_scrape_task( + task_id, + status="failed", + finished_at=datetime.now(timezone.utc), + error_message=str(exc)[:2000], + ) + logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc) + raise self.retry(exc=exc) + + +@shared_task( + name="iaai_scraper.worker.tasks.sync_listing_task", + bind=True, + max_retries=1, + default_retry_delay=120, + acks_late=True, +) +def sync_listing_task( + self, + make: str | None = None, + model: str | None = None, + lane: str = "iaai_cars", + limit: int | None = None, + only_new: bool | None = None, +): + task_id = self.request.id + persistence = _get_persistence() + persistence.create_tables() + + persistence.update_scrape_task( + task_id, + status="running", + started_at=datetime.now(timezone.utc), + ) + + try: + with IAAIScraper() as scraper: + result = scraper.sync_listing( + make=make, + model=model, + lane=lane, + limit=limit, + only_new=only_new, + ) + + summary = { + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + } + persistence.update_scrape_task( + task_id, + status="success", + finished_at=datetime.now(timezone.utc), + result_summary=json.dumps(summary, default=str), + ) + logger.info( + "sync_listing_task completed: %d upserted, %d failed", + summary["cars_upserted"], summary["cars_failed"], + ) + return {"status": "success", **summary} + + except Exception as exc: + persistence.update_scrape_task( + task_id, + status="failed", + finished_at=datetime.now(timezone.utc), + error_message=str(exc)[:2000], + ) + logger.error("sync_listing_task failed: %s", exc) + raise self.retry(exc=exc) diff --git a/main.py b/main.py new file mode 100644 index 0000000..2c0a646 --- /dev/null +++ b/main.py @@ -0,0 +1,5 @@ +from iaai_scraper.cli import main + + +if __name__ == "__main__": + main() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..3475ac5 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +addopts = -q --disable-warnings +python_files = tests/test_*.py +log_cli = false diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f91a842 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +playwright>=1.53.0 +python-dotenv>=1.0.1 +pydantic>=2.8.2 +SQLAlchemy>=2.0.32 +fastapi>=0.115.0 +uvicorn>=0.34.0 +celery>=5.4.0 +redis>=5.2.0 +psycopg2-binary>=2.9.9 +alembic>=1.14.0 +pytest>=8.3.0 +pytest-cov>=5.0.0 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b33f680 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +import logging + + +def pytest_configure() -> None: + logging.disable(logging.CRITICAL) diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..62ef84d --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from sqlalchemy import select + +from iaai_scraper.core.config import Settings +from iaai_scraper.storage.db import PersistenceService +from iaai_scraper.storage.models import Car, Image, SyncRun +from iaai_scraper.storage.schemas import CarRecord, ImageRecord + + +class TestPersistenceServiceIntegration(unittest.TestCase): + def setUp(self) -> None: + self.tmp_dir = tempfile.TemporaryDirectory() + db_path = Path(self.tmp_dir.name) / "test.sqlite" + + self.settings = Settings() + self.settings.database.url = f"sqlite:///{db_path.as_posix()}" + self.settings.database.echo = False + + self.persistence = PersistenceService(self.settings) + self.persistence.create_tables() + + def tearDown(self) -> None: + self.persistence.engine.dispose() + self.tmp_dir.cleanup() + + @staticmethod + def _record(origin_id: str, *, price: int = 1000, content_hash: str = "hash1") -> CarRecord: + return CarRecord( + parser_id=f"iaai:{origin_id}", + brand="Toyota", + model="Camry", + year=2014, + price=price, + origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + content_hash=content_hash, + images=[ + ImageRecord( + fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", + preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300", + order_index=0, + ) + ], + raw_attributes={"foo": "bar"}, + ) + + def test_insert_update_and_skip_flow(self) -> None: + first = self._record("777", price=1000, content_hash="same") + inserted = self.persistence.upsert_car(first) + self.assertEqual(inserted["action"], "inserted") + self.assertEqual(inserted["images_upserted"], 1) + + same = self._record("777", price=1000, content_hash="same") + updated_same = self.persistence.upsert_car(same) + self.assertEqual(updated_same["action"], "skipped") + self.assertEqual(updated_same["images_upserted"], 0) + + changed = self._record("777", price=1500, content_hash="changed") + updated = self.persistence.upsert_car(changed) + self.assertEqual(updated["action"], "updated") + self.assertEqual(updated["images_upserted"], 1) + + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].price, 1500) + self.assertEqual(len(images), 1) + + def test_update_replaces_old_images(self) -> None: + first = self._record("888", content_hash="v1") + self.persistence.upsert_car(first) + + second = self._record("888", content_hash="v2") + second.images = [ + ImageRecord( + fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633", + preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + order_index=0, + ) + ] + self.persistence.upsert_car(second) + + with self.persistence.session_scope() as session: + images = session.execute(select(Image)).scalars().all() + + self.assertEqual(len(images), 1) + self.assertIn("imageKeys=2", images[0].fullres_image) + + def test_same_content_hash_is_skipped(self) -> None: + first = self._record("999", content_hash="same-hash") + self.persistence.upsert_car(first) + + second = self._record("999", content_hash="same-hash") + result = self.persistence.upsert_car(second) + + self.assertEqual(result["action"], "skipped") + self.assertEqual(result["images_upserted"], 0) + + def test_persistence_ignores_non_db_fields(self) -> None: + record = self._record("1000", content_hash="schema-test") + record.raw_attributes = {"vin": "123"} + record.mapping_notes = ["note"] + + result = self.persistence.upsert_car(record) + + self.assertEqual(result["action"], "inserted") + with self.persistence.session_scope() as session: + car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one() + self.assertEqual(car.origin_id, "1000") + + def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: + first_run_id = self.persistence.start_sync_run("lane-a") + second_run_id = self.persistence.start_sync_run("lane-b") + + self.assertNotEqual(first_run_id, second_run_id) + + with self.persistence.session_scope() as session: + first = session.get(SyncRun, first_run_id) + second = session.get(SyncRun, second_run_id) + + self.assertEqual(first.status, "failed") + self.assertIsNotNone(first.finished_at) + self.assertEqual(second.status, "running") + + def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: + first = self._record("OLD-ID", content_hash="v1") + first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + self.persistence.upsert_car(first) + + second = self._record("NEW-ID", content_hash="v2") + second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + result = self.persistence.upsert_car(second) + + self.assertEqual(result["action"], "updated") + with self.persistence.session_scope() as session: + cars = session.execute(select(Car)).scalars().all() + + self.assertEqual(len(cars), 1) + self.assertEqual(cars[0].origin_id, "NEW-ID") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_listing.py b/tests/test_listing.py new file mode 100644 index 0000000..c3d16d0 --- /dev/null +++ b/tests/test_listing.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.browser.pace import HumanPacer +from iaai_scraper.core.config import Settings +from iaai_scraper.browser.listing import ListingCollector + + +class _FakePage: + def __init__(self, counts: dict[str, int]) -> None: + self._counts = counts + + class _Locator: + def __init__(self, count_value: int) -> None: + self._count_value = count_value + + def count(self) -> int: + return self._count_value + + def locator(self, selector: str) -> "_FakePage._Locator": + return _FakePage._Locator(self._counts.get(selector, 0)) + + +class TestListingUnit(unittest.TestCase): + def test_has_next_page_true_for_known_selector(self) -> None: + page = _FakePage({"a[aria-label*='Next']": 1}) + self.assertTrue(ListingCollector._has_next_page(page)) + + def test_has_next_page_false_when_no_selectors(self) -> None: + page = _FakePage({}) + self.assertFalse(ListingCollector._has_next_page(page)) + + def test_constructor_with_settings_and_pacer(self) -> None: + settings = Settings() + pacer = HumanPacer(settings) + collector = ListingCollector(settings, pacer) + self.assertIsNotNone(collector) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py new file mode 100644 index 0000000..93b9fd7 --- /dev/null +++ b/tests/test_mappers.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.parsing.mapper import CarMapper + + +class TestCarMapper(unittest.TestCase): + def setUp(self) -> None: + self.mapper = CarMapper() + + def test_content_hash_is_sha256(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US", + vehicle_summary={"make": "Toyota", "model": "Camry", "year": "2014"}, + payload_insights={ + "vehicle_core": {"odometer": "120,000", "body_type": "sedan"}, + "pricing": {"buy_now": "$4,500"}, + "damage": {"primary": "normal wear"}, + "auction": {}, + "images": {"urls": []}, + }, + ) + + self.assertEqual(len(record.content_hash), 64) + + def test_content_hash_changes_when_image_set_changes(self) -> None: + record_one = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US", + vehicle_summary={ + "make": "Toyota", + "model": "Camry", + "year": "2014", + "image_urls": ["https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633"], + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + record_two = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/45089484~US", + vehicle_summary={ + "make": "Toyota", + "model": "Camry", + "year": "2014", + "image_urls": ["https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633"], + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertNotEqual(record_one.content_hash, record_two.content_hash) + + def test_deduplicates_images_by_image_key(self) -> None: + urls = [ + "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", + "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + ] + + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(len(record.images), 2) + self.assertIn("width=845", record.images[0].fullres_image) + self.assertIn("height=633", record.images[0].fullres_image) + + def test_no_damage_marker_is_not_damaged(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_summary={"make": "Ford", "model": "Focus"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {}, + "damage": {"primary": "normal wear"}, + "auction": {}, + "images": {}, + }, + ) + + self.assertFalse(record.is_damaged) + + def test_unknown_and_empty_values_fallbacks(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/999~US", + vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "UNKNOWN") + self.assertEqual(record.model, "UNKNOWN") + self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None) + self.assertEqual(record.drive, "NA") + self.assertEqual(record.gearbox, "NA") + + def test_mapper_handles_case_and_spaces(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/888~US", + vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.drive, "FWD") + self.assertEqual(record.gearbox, "AT") + + def test_price_parsing_dirty_formats(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/777~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "USD 4,500 - 5,200"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + + self.assertEqual(record.price, 5200) + + def test_currency_detection_from_symbol(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://www.iaai.com/VehicleDetail/778~US", + vehicle_summary={"make": "Toyota", "model": "Corolla"}, + payload_insights={ + "vehicle_core": {}, + "pricing": {"buy_now": "€4.500,00"}, + "damage": {}, + "auction": {}, + "images": {}, + }, + ) + + self.assertEqual(record.currency, "EUR") + self.assertEqual(record.price, 4500) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..1b9f25b --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.parsing.parser import VehicleParser + + +class TestVehicleParserUnit(unittest.TestCase): + def setUp(self) -> None: + self.parser = VehicleParser() + + def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None: + dom_text = """ + Stock #: + 45089484 + VIN (Status): + 1HGCM82633A123456 (OK) + Primary Damage: + Front End + """ + result = self.parser._parse_dom_key_value_pairs(dom_text) + self.assertEqual(result.get("lot_number"), "45089484") + self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)") + self.assertEqual(result.get("primary_damage"), "Front End") + + def test_parse_title_for_year_make_model(self) -> None: + parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") + self.assertEqual(parsed["year"], "2014") + self.assertEqual(parsed["make"], "TOYOTA") + self.assertEqual(parsed["model"], "CAMRY") + + def test_extract_image_urls_filters_other_vehicle(self) -> None: + vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" + payloads = [ + { + "imageUrls": [ + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", + ] + } + ] + urls = self.parser._extract_image_urls(payloads, "", vehicle_url) + self.assertEqual(len(urls), 1) + self.assertIn("45089484", urls[0]) + + def test_extract_image_urls_deduplicates_same_url(self) -> None: + vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" + payloads = [{"imageUrls": [ + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + ]}] + urls = self.parser._extract_image_urls(payloads, "", vehicle_url) + self.assertEqual(len(urls), 1) + + def test_dom_hints_detect_captcha_and_antibot(self) -> None: + hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") + self.assertTrue(hints["has_captcha_text"]) + self.assertTrue(hints["has_antibot_text"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py new file mode 100644 index 0000000..c9a1d7f --- /dev/null +++ b/tests/test_scraper.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock + +from iaai_scraper.core.config import Settings +from iaai_scraper.scraper import IAAIScraper +from iaai_scraper.storage.schemas import CarRecord + + +def make_db_record(origin_id: str) -> dict[str, object]: + return CarRecord( + parser_id=f"iaai:{origin_id}", + brand="Toyota", + model="Camry", + origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_id=origin_id, + slug=f"toyota-camry-{origin_id}", + ).model_dump(mode="json") + + +class TestScraperSync(unittest.TestCase): + def _make_scraper(self) -> IAAIScraper: + s = Settings() + s.log_level = "CRITICAL" + s.database.url = "sqlite://" + return IAAIScraper(s) + + def test_sync_vehicle_uses_db_record_without_remapping(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=1) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) + + scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) + scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) + + result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") + + self.assertEqual(result["status"], "success") + self.assertIn("trace_id", result) + self.assertIn("elapsed_seconds", result) + self.assertEqual(scraper.persistence.upsert_car.call_count, 1) + + def test_sync_listing_uses_db_record_without_remapping(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=2) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) + + scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]}) + page = MagicMock() + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")}) + scraper._get_page = MagicMock(return_value=page) + scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) + + result = scraper.sync_listing() + + self.assertEqual(result["cars_upserted"], 1) + self.assertEqual(result["cars_failed"], 0) + self.assertIn("trace_id", result) + self.assertIn("elapsed_seconds", result) + self.assertEqual(scraper.persistence.upsert_car.call_count, 1) + page.close.assert_called_once() + + def test_sync_listing_respects_limit(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=3) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/VehicleDetail/333~US", + ] + }) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")}) + scraper._get_page = MagicMock(return_value=MagicMock()) + + scraper.sync_listing(limit=1) + + self.assertEqual(scraper._scrape_on_page.call_count, 1) + + def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=4) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) + scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]}) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")}) + scraper._get_page = MagicMock(return_value=MagicMock()) + + result = scraper.sync_listing() + + self.assertEqual(result["cars_upserted"], 0) + + def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None: + scraper = self._make_scraper() + + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=5) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) + scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"}) + scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"}) + + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.iaai.com/VehicleDetail/111~US", # exists by URL + "https://www.iaai.com/VehicleDetail/222~US", # exists by ID + "https://www.iaai.com/VehicleDetail/333~US", # new + ] + }) + page = MagicMock() + scraper._get_page = MagicMock(return_value=page) + scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")}) + + result = scraper.sync_listing(only_new=True) + + self.assertEqual(result["skipped_existing"], 2) + self.assertEqual(result["cars_upserted"], 1) + self.assertEqual(scraper._scrape_on_page.call_count, 1) + scraper.persistence.get_existing_origin_urls.assert_called_once() + scraper.persistence.get_existing_origin_ids.assert_called_once() + + def test_close_resets_browser_state(self) -> None: + scraper = self._make_scraper() + scraper.context = MagicMock() + scraper.browser = MagicMock() + scraper.playwright = MagicMock() + + scraper.close() + + self.assertIsNone(scraper.context) + self.assertIsNone(scraper.browser) + self.assertIsNone(scraper.playwright) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..fff40fa --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import unittest + +from iaai_scraper.core.utils import deep_find_key + + +class TestDeepFindKey(unittest.TestCase): + def test_finds_key_in_nested_structure(self) -> None: + payload = { + "root": { + "target": "a", + "nested": [{"target": "b"}, {"x": 1}], + } + } + + found = deep_find_key(payload, {"target"}) + self.assertEqual(found, ["a", "b"]) + + def test_respects_max_depth(self) -> None: + payload = {"l1": {"l2": {"l3": {"target": "value"}}}} + + found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2) + found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8) + + self.assertEqual(found_too_shallow, []) + self.assertEqual(found_enough_depth, ["value"]) + + +if __name__ == "__main__": + unittest.main()