initial openlane project
This commit is contained in:
qananasikq
@@ -1,27 +1,24 @@
|
||||
x-app-env: &app-env
|
||||
# NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
|
||||
IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
|
||||
IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
|
||||
# PostgreSQL — единственная БД в Docker
|
||||
OPENLANE_DATABASE_URL: postgresql+psycopg2://openlane:openlane@postgres:5432/openlane_scraper
|
||||
OPENLANE_REDIS_URL: ${OPENLANE_REDIS_URL:-redis://redis:6379/0}
|
||||
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
|
||||
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
|
||||
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
|
||||
OPENLANE_DATABASE_POOL_RECYCLE_SECONDS: ${OPENLANE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
|
||||
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
|
||||
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
|
||||
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
|
||||
# 0 => без лимита (в коде интерпретируется как None).
|
||||
# После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
|
||||
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
|
||||
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
|
||||
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
|
||||
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
|
||||
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
|
||||
IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
|
||||
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
|
||||
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
|
||||
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
|
||||
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
|
||||
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
|
||||
IAAI_BROWSER_ENGINE: chromium
|
||||
OPENLANE_MAX_CARS_LIMIT: ${OPENLANE_MAX_CARS_LIMIT:-20}
|
||||
OPENLANE_THROTTLE_MIN_SECONDS: ${OPENLANE_THROTTLE_MIN_SECONDS:-2.0}
|
||||
OPENLANE_THROTTLE_MAX_SECONDS: ${OPENLANE_THROTTLE_MAX_SECONDS:-5.0}
|
||||
OPENLANE_CONCURRENCY: ${OPENLANE_CONCURRENCY:-1}
|
||||
OPENLANE_MAX_PAGES: ${OPENLANE_MAX_PAGES:-512}
|
||||
OPENLANE_BROWSER_ENGINE: chromium
|
||||
OPENLANE_HEADLESS: "true"
|
||||
OPENLANE_STORAGE_STATE_FILE: /data/openlane/openlane_storage_state.json
|
||||
OPENLANE_RUNTIME_CONFIG_FILE: /app/runtime_config.json
|
||||
TZ: ${TZ:-UTC}
|
||||
|
||||
x-env-file: &env-file
|
||||
@@ -40,23 +37,24 @@ x-worker-service: &worker-service
|
||||
volumes:
|
||||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||||
- tokens_data:/data
|
||||
- openlane_data:/data/openlane
|
||||
|
||||
services:
|
||||
# PostgreSQL
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: iaai-postgres
|
||||
container_name: openlane-postgres
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: iaai
|
||||
POSTGRES_PASSWORD: iaai
|
||||
POSTGRES_DB: iaai_scraper
|
||||
POSTGRES_USER: openlane
|
||||
POSTGRES_PASSWORD: openlane
|
||||
POSTGRES_DB: openlane_scraper
|
||||
ports:
|
||||
- "127.0.0.1:5432:5432"
|
||||
- "127.0.0.1:${OPENLANE_POSTGRES_PORT:-55432}:5432"
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
|
||||
test: ["CMD-SHELL", "pg_isready -U openlane -d openlane_scraper"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
@@ -69,10 +67,10 @@ services:
|
||||
# Redis (Celery broker)
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: iaai-redis
|
||||
container_name: openlane-redis
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:6379:6379"
|
||||
- "127.0.0.1:${OPENLANE_REDIS_PORT:-56379}:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
@@ -93,7 +91,7 @@ services:
|
||||
# DB migrations
|
||||
migrate:
|
||||
<<: *app-service
|
||||
container_name: iaai-migrate
|
||||
container_name: openlane-migrate
|
||||
restart: "no"
|
||||
depends_on:
|
||||
postgres:
|
||||
@@ -103,17 +101,17 @@ services:
|
||||
# FastAPI
|
||||
api:
|
||||
<<: *app-service
|
||||
container_name: iaai-api
|
||||
container_name: openlane-api
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:8000:8000"
|
||||
- "127.0.0.1:${OPENLANE_API_PORT:-58000}:8000"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
command: >
|
||||
uvicorn iaai_scraper.api.app:app
|
||||
uvicorn openlane_scraper.api.app:app
|
||||
--host 0.0.0.0 --port 8000 --workers 1
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
|
||||
@@ -131,11 +129,11 @@ services:
|
||||
# Celery Worker
|
||||
worker:
|
||||
<<: *worker-service
|
||||
container_name: iaai-worker
|
||||
container_name: openlane-worker
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
|
||||
memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
|
||||
mem_limit: ${OPENLANE_WORKER_MEM_LIMIT:-2g}
|
||||
memswap_limit: ${OPENLANE_WORKER_MEM_LIMIT:-2g}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
@@ -143,7 +141,7 @@ services:
|
||||
condition: service_healthy
|
||||
stop_grace_period: 60s
|
||||
command: >
|
||||
celery -A iaai_scraper.worker.celery_app worker
|
||||
celery -A openlane_scraper.worker.celery_app worker
|
||||
--loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
|
||||
--pidfile=/tmp/celery-worker.pid
|
||||
-Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
|
||||
@@ -162,7 +160,7 @@ services:
|
||||
# Celery Beat (периодический планировщик)
|
||||
beat:
|
||||
<<: *worker-service
|
||||
container_name: iaai-beat
|
||||
container_name: openlane-beat
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
depends_on:
|
||||
@@ -171,7 +169,7 @@ services:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
command: >
|
||||
celery -A iaai_scraper.worker.celery_app beat
|
||||
celery -A openlane_scraper.worker.celery_app beat
|
||||
--loglevel=info
|
||||
--pidfile=/tmp/celery-beat.pid
|
||||
--schedule=/tmp/celerybeat-schedule
|
||||
@@ -187,7 +185,17 @@ services:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
openlane-auth:
|
||||
<<: *worker-service
|
||||
container_name: openlane-auth
|
||||
profiles: ["auth"]
|
||||
stdin_open: true
|
||||
tty: true
|
||||
command: >
|
||||
python scripts/explore_site.py
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
tokens_data:
|
||||
openlane_data:
|
||||
|
||||
Reference in New Issue
Block a user