add project setup

2026-04-21 23:48:20 +03:00
commit 6b55c8cb28
11 changed files with 1792 additions and 0 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -0,0 +1,23 @@
+__pycache__/
+.pytest_cache/
+.venv/
+venv/
+
+.coverage
+coverage.xml
+
+*.pyc
+*.pyo
+*.pyd
+*.log
+*.db
+
+.env
+.git/
+.vscode/
+
+*.egg-info/
+dist/
+build/
+artifacts/
+tokens_data/
--- a/.env.example
+++ b/.env.example
@@ -0,0 +1,62 @@
+# --- General ---
+OPENLANE_HEADLESS=true
+OPENLANE_LOG_LEVEL=INFO
+OPENLANE_TIMEOUT_MS=45000
+
+# --- OpenLane Auth & API ---
+OPENLANE_USERNAME=
+OPENLANE_PASSWORD=
+OPENLANE_SIGN_IN_URL=https://app.openlane.com/sign_in
+OPENLANE_SEARCH_URL=https://app.openlane.com/search?tab=marketplace
+OPENLANE_API_SEARCH_URL=https://app.openlane.com/api/v4/search
+OPENLANE_SOURCE_TAB=marketplace
+OPENLANE_SALE_TYPES=marketplace
+OPENLANE_PAGE_SIZE=30
+OPENLANE_MAX_PAGES=512
+OPENLANE_MAX_CARS_LIMIT=20
+OPENLANE_THROTTLE_MIN_SECONDS=0.3
+OPENLANE_THROTTLE_MAX_SECONDS=0.8
+OPENLANE_RETRY_SCHEDULE_SECONDS=2,5,10
+OPENLANE_REQUEST_TIMEOUT_MS=45000
+OPENLANE_REFRESH_TOKEN=
+OPENLANE_OKTA_CLIENT_ID=
+OPENLANE_OKTA_TOKEN_ENDPOINT=https://okta.iam.karglobal.com/oauth2/default/v1/token
+OPENLANE_OKTA_REDIRECT_URI=https://app.openlane.com/sign_in
+OPENLANE_OKTA_TIMEOUT_SECONDS=15
+
+# --- Output & State ---
+OPENLANE_OUTPUT_DIR=artifacts/openlane
+OPENLANE_JSONL_OUTPUT=artifacts/openlane/openlane_search.jsonl
+OPENLANE_AGGREGATED_OUTPUT=artifacts/openlane/openlane_search_aggregated.json
+OPENLANE_CHECKPOINT_FILE=artifacts/openlane/openlane_checkpoint.json
+OPENLANE_STORAGE_STATE_FILE=/data/openlane/openlane_storage_state.json
+OPENLANE_PERSIST_STORAGE_STATE=true
+
+# --- Database (PostgreSQL) ---
+OPENLANE_DATABASE_URL=postgresql+psycopg2://openlane:openlane@postgres:5432/openlane_scraper
+OPENLANE_DATABASE_ECHO=false
+OPENLANE_DATABASE_POOL_SIZE=5
+OPENLANE_DATABASE_MAX_OVERFLOW=5
+
+# --- Redis ---
+OPENLANE_REDIS_URL=redis://redis:6379/0
+
+# --- Celery ---
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CELERY_TASK_SOFT_TIME_LIMIT=86400
+CELERY_TASK_TIME_LIMIT=86520
+CELERY_BROKER_VISIBILITY_TIMEOUT=90000
+CELERY_WORKER_CONCURRENCY=1
+CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_BEAT_SYNC_LIMIT=0
+
+# --- Docker host ports (override if occupied) ---
+OPENLANE_API_PORT=58000
+OPENLANE_POSTGRES_PORT=55432
+OPENLANE_REDIS_PORT=56379
+
+# --- Proxy (optional) ---
+# OPENLANE_PROXY_SERVER=http://127.0.0.1:8899
+# OPENLANE_PROXY_USERNAME=
+# OPENLANE_PROXY_PASSWORD=
--- a/.gitattributes
+++ b/.gitattributes
@@ -0,0 +1,6 @@
+# Force LF for shell scripts (critical for Docker on Windows)
+*.sh text eol=lf
+entrypoint.sh text eol=lf
+
+# Default for other files
+* text=auto
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,30 @@
+__pycache__/
+.pytest_cache/
+.coverage
+coverage.xml
+.env*
+!.env.example
+.venv/
+venv/
+*.pyc
+*.pyo
+*.pyd
+*.log
+*.db
+storage_state.json
+.vscode/
+*.egg-info/
+dist/
+build/
+artifacts/
+celerybeat-schedule*
+tokens_data/
+tokens.json
+
+# Ad-hoc debug scripts and one-off downloads
+_*.sh
+_*.tar.gz
+burst_*.ps1
+burst_*.txt
+sitemap*.xml
+vd_*.html
--- a/34
+++ b/34
@@ -0,0 +1,34 @@
+FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+RUN groupadd --system app && useradd --system --gid app --create-home app
+
+WORKDIR /app
+
+COPY pyproject.toml uv.lock ./
+RUN pip install --no-cache-dir uv \
+    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
+    && pip install --no-cache-dir -r /tmp/requirements.txt \
+    && pip install --no-cache-dir playwright-stealth
+
+# orjson + brotli для ускорения JSON/HTTP.
+RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0"
+
+# Chromium + Firefox.
+RUN python -m playwright install chromium firefox
+
+COPY . .
+RUN pip install --no-cache-dir -e .
+RUN python -m compileall -q openlane_scraper
+RUN chmod +x entrypoint.sh
+RUN chown -R app:app /app
+
+STOPSIGNAL SIGINT
+
+USER app
+
+# По умолчанию  API, но worker/beat переопределяют CMD в docker-compose
+ENTRYPOINT ["./entrypoint.sh"]
+CMD ["uvicorn", "openlane_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
--- a/README.md
+++ b/README.md
@@ -0,0 +1,152 @@
+# OpenLane Scraper
+
+Парсер автомобилей с [OpenLane marketplace](https://app.openlane.com). Забирает данные через API (`GET /api/v4/search`), сохраняет в PostgreSQL. Работает через Playwright (для авторизации), FastAPI, Celery и Docker.
+
+## Архитектура
+
+- **Авторизация**: Okta OAuth2 PKCE через `okta.iam.karglobal.com`. Сессия сохраняется в `storage_state.json` и переиспользуется между запусками.
+- **Сбор данных**: API-запросы с заголовками `x-rbz-user-id`, `x-rbz-dealership-id` (извлекаются из JWT access_token).
+- **Защита аккаунта**: throttle 2–5s между запросами, jitter 0–120s перед стартом, circuit breaker (2 ошибки подряд → стоп), 403→немедленный стоп, 429→backoff×3.
+- **Anti-detection**: блокировка ресурсов (bugsnag, intercom, pusher, analytics), stealth patches.
+
+## Локальный запуск
+
+### Требования
+
+- **Python 3.11+**
+- **PostgreSQL** и **Redis** (для Celery)
+
+### Быстрый старт
+
+```bash
+git clone <repo-url>
+cd openlane_scraper_project
+pip install -e .
+playwright install chromium
+
+cp .env.example .env
+# Заполнить OPENLANE_USERNAME, OPENLANE_PASSWORD в .env
+
+alembic upgrade head
+```
+
+### Запуск
+
+```bash
+# Терминал 1 — API
+uvicorn openlane_scraper.api.app:app --reload --port 8000
+
+# Терминал 2 — Celery Worker
+celery -A openlane_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo -Q scraping
+
+# Терминал 3 — Celery Beat (каждый час)
+celery -A openlane_scraper.worker.celery_app beat --loglevel=info
+```
+
+CLI:
+
+```bash
+openlane scrape-openlane --limit 20
+```
+
+## Docker
+
+```bash
+cp .env.example .env
+docker compose up -d
+```
+
+Сервисы: `postgres`, `redis`, `migrate`, `api`, `worker`, `beat`.
+
+
+- `beat` запускает сбор **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`)
+- Лимит по умолчанию: 20 машин (`OPENLANE_MAX_CARS_LIMIT=20`)
+
+API (Docker): `http://localhost:58000` · Swagger: `http://localhost:58000/docs`
+
+### Первый логин (storage state)
+
+```bash
+docker compose run --rm --service-ports openlane-auth openlane openlane-login
+# Войти вручную в браузере — storage state сохранится в volume
+```
+
+## API
+
+- `GET /health` — статус сервиса
+- `GET /api/v1/stats` — статистика (машины, картинки, топ брендов)
+- `GET /api/v1/cars` — список с пагинацией
+- `GET /api/v1/cars/{id}` — карточка с картинками
+- `POST /api/v1/tasks/sync-listing` — запустить сбор
+- `GET /api/v1/sync-runs` — история запусков
+
+## Структура
+
+```text
+openlane_scraper/
+  scraper.py              — оркестратор: circuit breaker, batched upsert, early-stop
+  cli.py                  — CLI-команды (scrape-openlane, openlane-login)
+
+  browser/
+    factory.py            — создание браузера, stealth-инъекции, resource blocking
+
+  openlane/
+    auth.py               — авторизация через storage_state / Okta PKCE
+    client.py             — запросы к /api/v4/search с JWT-заголовками
+    checkpoint.py         — checkpoint/resume по страницам
+    writer.py             — JSONL + aggregated JSON
+    runner.py             — orchestration, retry/backoff, progress
+
+  storage/
+    models.py             — SQLAlchemy: Car, Image, SyncRun
+    schemas.py            — Pydantic: CarRecord, ImageRecord, CarRead
+    enums.py              — enum-значения (drive, gearbox, body_type, ...)
+    db.py                 — PersistenceService: upsert, sync runs, статистика
+
+  api/
+    app.py                — FastAPI factory
+    deps.py               — dependency injection
+    routes/               — health, cars, tasks
+
+  worker/
+    celery_app.py         — конфиг Celery, beat-расписание
+    tasks.py              — sync_listing_task с jitter и lock
+
+  core/
+    config.py             — Settings (dataclass), env-переменные
+    logs.py               — логирование с trace_id
+    retry.py              — @retryable с exponential backoff
+    runtime_config.py     — runtime-конфиг
+    utils.py              — утилиты
+```
+
+## Конфигурация
+
+Всё через env-переменные (полный список в `.env.example`):
+
+- **OpenLane**: `OPENLANE_USERNAME`, `OPENLANE_PASSWORD`, `OPENLANE_MAX_CARS_LIMIT`
+- **БД**: `OPENLANE_DATABASE_URL`, `OPENLANE_DATABASE_POOL_SIZE`
+- **Redis**: `OPENLANE_REDIS_URL`
+- **Celery**: `CELERY_BROKER_URL`, `CELERY_BEAT_SYNC_INTERVAL_MINUTES`
+- **Прокси**: `OPENLANE_PROXY_SERVER`, `OPENLANE_PROXY_USERNAME`, `OPENLANE_PROXY_PASSWORD`
+
+## Миграции
+
+В Docker миграции выполняются сервисом `migrate` (`alembic upgrade head`).
+
+```bash
+alembic upgrade head
+alembic revision --autogenerate -m "add_column_x"
+```
+
+## Тесты
+
+```bash
+pytest -q
+```
+
+Тесты работают на SQLite in-memory, без внешних зависимостей.
+
+## Runtime-фильтры
+
+Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -0,0 +1,201 @@
+x-app-env: &app-env
+  # PostgreSQL — единственная БД в Docker
+  OPENLANE_DATABASE_URL: postgresql+psycopg2://openlane:openlane@postgres:5432/openlane_scraper
+  OPENLANE_REDIS_URL: ${OPENLANE_REDIS_URL:-redis://redis:6379/0}
+  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
+  OPENLANE_DATABASE_POOL_RECYCLE_SECONDS: ${OPENLANE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
+  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+  OPENLANE_MAX_CARS_LIMIT: ${OPENLANE_MAX_CARS_LIMIT:-20}
+  OPENLANE_THROTTLE_MIN_SECONDS: ${OPENLANE_THROTTLE_MIN_SECONDS:-2.0}
+  OPENLANE_THROTTLE_MAX_SECONDS: ${OPENLANE_THROTTLE_MAX_SECONDS:-5.0}
+  OPENLANE_CONCURRENCY: ${OPENLANE_CONCURRENCY:-1}
+  OPENLANE_MAX_PAGES: ${OPENLANE_MAX_PAGES:-512}
+  OPENLANE_BROWSER_ENGINE: chromium
+  OPENLANE_HEADLESS: "true"
+  OPENLANE_STORAGE_STATE_FILE: /data/openlane/openlane_storage_state.json
+  OPENLANE_RUNTIME_CONFIG_FILE: /app/runtime_config.json
+  TZ: ${TZ:-UTC}
+
+x-env-file: &env-file
+  - path: .env
+    required: false
+
+x-app-service: &app-service
+  build: .
+  env_file: *env-file
+  environment: *app-env
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+
+x-worker-service: &worker-service
+  <<: *app-service
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+    - tokens_data:/data
+    - openlane_data:/data/openlane
+
+services:
+  # PostgreSQL
+  postgres:
+    image: postgres:16-alpine
+    container_name: openlane-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: openlane
+      POSTGRES_PASSWORD: openlane
+      POSTGRES_DB: openlane_scraper
+    ports:
+      - "127.0.0.1:${OPENLANE_POSTGRES_PORT:-55432}:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U openlane -d openlane_scraper"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # Redis (Celery broker)
+  redis:
+    image: redis:7-alpine
+    container_name: openlane-redis
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:${OPENLANE_REDIS_PORT:-56379}:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+    command: >
+      redis-server
+      --appendonly yes
+      --save 60 1000
+    volumes:
+      - redisdata:/data
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # DB migrations
+  migrate:
+    <<: *app-service
+    container_name: openlane-migrate
+    restart: "no"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    command: alembic upgrade head
+
+  # FastAPI
+  api:
+    <<: *app-service
+    container_name: openlane-api
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:${OPENLANE_API_PORT:-58000}:8000"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    command: >
+      uvicorn openlane_scraper.api.app:app
+      --host 0.0.0.0 --port 8000 --workers 1
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    stop_grace_period: 30s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # Celery Worker
+  worker:
+    <<: *worker-service
+    container_name: openlane-worker
+    restart: unless-stopped
+    init: true
+    mem_limit: ${OPENLANE_WORKER_MEM_LIMIT:-2g}
+    memswap_limit: ${OPENLANE_WORKER_MEM_LIMIT:-2g}
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    stop_grace_period: 60s
+    command: >
+      celery -A openlane_scraper.worker.celery_app worker
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
+      --pidfile=/tmp/celery-worker.pid
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # Celery Beat (периодический планировщик)
+  beat:
+    <<: *worker-service
+    container_name: openlane-beat
+    restart: unless-stopped
+    init: true
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    command: >
+      celery -A openlane_scraper.worker.celery_app beat
+      --loglevel=info
+      --pidfile=/tmp/celery-beat.pid
+      --schedule=/tmp/celerybeat-schedule
+    healthcheck:
+      test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  openlane-auth:
+    <<: *worker-service
+    container_name: openlane-auth
+    profiles: ["auth"]
+    stdin_open: true
+    tty: true
+    command: >
+      python scripts/explore_site.py
+
+volumes:
+  pgdata:
+  redisdata:
+  tokens_data:
+  openlane_data:
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+set -euo pipefail
+
+exec "$@"
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "openlane-scraper"
+version = "0.1.0"
+description = "OpenLane scraper service with FastAPI, Celery, Playwright and PostgreSQL"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+  "alembic>=1.14.0",
+  "celery>=5.4.0",
+  "fastapi>=0.115.0",
+  "playwright>=1.53.0",
+  "psycopg2-binary>=2.9.9",
+  "pydantic>=2.8.2",
+  "python-dotenv>=1.0.1",
+  "redis>=5.2.0",
+  "sqlalchemy>=2.0.32",
+  "uvicorn>=0.34.0",
+  "urllib3>=2.0.0",
+  "orjson>=3.10.0",
+  "brotli>=1.1.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8.3.0",
+  "pytest-cov>=5.0.0",
+]
+
+[project.scripts]
+openlane = "openlane_scraper.cli:main"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+include = ["openlane_scraper*"]
+
+[tool.pytest.ini_options]
+addopts = "-q --disable-warnings"
+python_files = ["tests/test_*.py"]
+log_cli = false
--- a/runtime_config.json
+++ b/runtime_config.json
@@ -0,0 +1,104 @@
+{
+  "sync": {
+    "name": null,
+    "ids_initial_size": null,
+    "ids_next_size": null,
+    "ids_max_pages": null,
+    "condition_check_enabled": false,
+    "lane": "openlane_marketplace",
+    "only_new": false,
+    "limit": null
+  },
+  "filters": {
+    "price": {
+      "min": null,
+      "max": null
+    },
+    "mileage": {
+      "min": null,
+      "max": null
+    },
+    "flags": {
+      "damaged_only": null,
+      "run_and_drive": null
+    },
+    "brands": [
+      "Acura",
+      "Alfa Romeo",
+      "Alpina",
+      "Aston Martin",
+      "Audi",
+      "BMW",
+      "BMW ALPINA",
+      "Bentley",
+      "Bugatti",
+      "Cadillac",
+      "Caterham",
+      "Chevrolet",
+      "Chrysler",
+      "Cupra",
+      "Daimler",
+      "Dodge",
+      "Ferrari",
+      "Ford",
+      "Genesis",
+      "Honda",
+      "Hummer",
+      "Hyundai",
+      "Infiniti",
+      "Isuzu",
+      "Jaguar",
+      "Jeep",
+      "Kia",
+      "Lamborghini",
+      "Land Rover",
+      "Lexus",
+      "Lincoln",
+      "Lotus",
+      "Lucid",
+      "Maserati",
+      "Maybach",
+      "Mazda",
+      "McLaren",
+      "Mercedes-Benz",
+      "Mercury",
+      "Mini",
+      "Mitsubishi",
+      "Nissan",
+      "Oldsmobile",
+      "Pagani",
+      "Plymouth",
+      "Pontiac",
+      "Porsche",
+      "Ram",
+      "Ravon",
+      "Rivian",
+      "Rolls-Royce",
+      "Rolls Royce",
+      "Rover",
+      "Saturn",
+      "Skoda",
+      "Smart",
+      "Subaru",
+      "Suzuki",
+      "Tesla",
+      "Toyota",
+      "Volkswagen",
+      "Volvo",
+      "KTM AG",
+      "Koenigsegg",
+      "Rimac"
+    ],
+    "models": [],
+    "years": [],
+    "body_types": [],
+    "colors": [],
+    "drives": [],
+    "gearboxes": [],
+    "locations": [],
+    "exclude_brands": [],
+    "exclude_models": [],
+    "exclude_years": [],
+    "exclude_body_types": []
+  }
+}
--- a/uv.lock
+++ b/uv.lock