219 lines
7.0 KiB
YAML
219 lines
7.0 KiB
YAML
x-app-env: &app-env
|
||
# Всегда используем Postgres.
|
||
IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
|
||
IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
|
||
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
|
||
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
|
||
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
|
||
IAAI_DATABASE_POOL_SIZE: ${IAAI_DATABASE_POOL_SIZE:-20}
|
||
IAAI_DATABASE_MAX_OVERFLOW: ${IAAI_DATABASE_MAX_OVERFLOW:-40}
|
||
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
|
||
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
|
||
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
|
||
IAAI_PROFILE: ${IAAI_PROFILE:-fast}
|
||
IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}}
|
||
IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true}
|
||
IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false}
|
||
IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false}
|
||
IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1}
|
||
IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-2}
|
||
IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0}
|
||
IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1}
|
||
IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
|
||
# 0 = без лимита.
|
||
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
|
||
CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
|
||
CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
|
||
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
|
||
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000}
|
||
IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
|
||
IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
|
||
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
|
||
IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-32}
|
||
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
|
||
IAAI_FILTERED_SEARCH_URL: ${IAAI_FILTERED_SEARCH_URL:-}
|
||
IAAI_FILTERED_SEARCH_URLS: ${IAAI_FILTERED_SEARCH_URLS:-}
|
||
IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-runtime}
|
||
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
|
||
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
|
||
IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
|
||
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
|
||
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json}
|
||
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
|
||
IAAI_BROWSER_ENGINE: chromium
|
||
# Self-heal для worker.
|
||
IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
|
||
IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
|
||
IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
|
||
IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
|
||
IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
|
||
# Если в Postgres нет записей дольше 60 минут при активной работе — полный restart с segment 1.
|
||
IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
|
||
TZ: ${TZ:-UTC}
|
||
|
||
x-env-file: &env-file
|
||
- path: .env
|
||
required: false
|
||
|
||
x-app-service: &app-service
|
||
build: .
|
||
env_file: *env-file
|
||
environment: *app-env
|
||
volumes:
|
||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||
|
||
x-worker-service: &worker-service
|
||
<<: *app-service
|
||
volumes:
|
||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||
- tokens_data:/data
|
||
|
||
services:
|
||
# PostgreSQL.
|
||
postgres:
|
||
image: postgres:16-alpine
|
||
container_name: iaai-postgres
|
||
restart: unless-stopped
|
||
environment:
|
||
POSTGRES_USER: iaai
|
||
POSTGRES_PASSWORD: iaai
|
||
POSTGRES_DB: iaai_scraper
|
||
ports:
|
||
- "127.0.0.1:5432:5432"
|
||
volumes:
|
||
- pgdata:/var/lib/postgresql/data
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 5
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
# Redis.
|
||
redis:
|
||
image: redis:7-alpine
|
||
container_name: iaai-redis
|
||
restart: unless-stopped
|
||
ports:
|
||
- "127.0.0.1:6379:6379"
|
||
healthcheck:
|
||
test: ["CMD", "redis-cli", "ping"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 5
|
||
command: >
|
||
redis-server
|
||
--appendonly yes
|
||
--save 60 1000
|
||
volumes:
|
||
- redisdata:/data
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
# Миграции.
|
||
migrate:
|
||
<<: *app-service
|
||
container_name: iaai-migrate
|
||
restart: "no"
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
command: alembic upgrade head
|
||
|
||
# API.
|
||
api:
|
||
<<: *app-service
|
||
container_name: iaai-api
|
||
restart: unless-stopped
|
||
ports:
|
||
- "127.0.0.1:8000:8000"
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
redis:
|
||
condition: service_healthy
|
||
command: >
|
||
uvicorn iaai_scraper.api.app:app
|
||
--host 0.0.0.0 --port 8000 --workers 1
|
||
healthcheck:
|
||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
|
||
interval: 30s
|
||
timeout: 10s
|
||
retries: 3
|
||
start_period: 40s
|
||
stop_grace_period: 30s
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
# Worker.
|
||
worker:
|
||
<<: *worker-service
|
||
restart: unless-stopped
|
||
init: true
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
redis:
|
||
condition: service_healthy
|
||
stop_grace_period: 60s
|
||
command: >
|
||
celery -A iaai_scraper.worker.celery_app worker
|
||
--loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork}
|
||
--pidfile=/tmp/celery-worker.pid
|
||
-Q iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
|
||
healthcheck:
|
||
# Проверка worker.
|
||
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
|
||
interval: 60s
|
||
timeout: 10s
|
||
retries: 3
|
||
start_period: 90s
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
# Beat.
|
||
beat:
|
||
<<: *worker-service
|
||
container_name: iaai-beat
|
||
restart: unless-stopped
|
||
init: true
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
redis:
|
||
condition: service_healthy
|
||
command: >
|
||
celery -A iaai_scraper.worker.celery_app beat
|
||
--loglevel=info
|
||
--pidfile=/tmp/celery-beat.pid
|
||
--schedule=/tmp/celerybeat-schedule
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"]
|
||
interval: 60s
|
||
timeout: 10s
|
||
retries: 3
|
||
start_period: 90s
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
volumes:
|
||
pgdata:
|
||
redisdata:
|
||
tokens_data:
|