222 lines
7.4 KiB
YAML
222 lines
7.4 KiB
YAML
x-app-env: &app-env
|
|
# Всегда используем Postgres.
|
|
IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
|
|
IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
|
|
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
|
|
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
|
|
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
|
|
# Для больших Search-выборок (десятки тысяч лотов) run должен жить дольше одного батча.
|
|
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-7200}
|
|
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-7500}
|
|
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-10800}
|
|
# 0 = без лимита.
|
|
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
|
|
# Следующий плановый прогон — не раньше чем через час после предыдущего tick beat.
|
|
CELERY_BEAT_SYNC_INTERVAL_MINUTES: ${CELERY_BEAT_SYNC_INTERVAL_MINUTES:-60}
|
|
# Любой внутренний follow-up после неполного bootstrap тоже не стартует сразу.
|
|
IAAI_SYNC_FOLLOWUP_MIN_DELAY_SECONDS: ${IAAI_SYNC_FOLLOWUP_MIN_DELAY_SECONDS:-3600}
|
|
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
|
|
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1500}
|
|
IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
|
|
# Стабильный fast-профиль: не душим IAAI слишком большим числом HTTPS detail-соединений.
|
|
IAAI_SCRAPING_PROFILE: fast
|
|
IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-96}
|
|
IAAI_MAX_RETRIES: ${IAAI_MAX_RETRIES:-2}
|
|
IAAI_RETRY_DELAY_SECONDS: ${IAAI_RETRY_DELAY_SECONDS:-0.35}
|
|
IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1}
|
|
CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-false}
|
|
CELERY_WORKER_CONCURRENCY: 4
|
|
CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
|
|
IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
|
|
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
|
|
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
|
|
IAAI_FAST_PATH_TIMEOUT_MS: ${IAAI_FAST_PATH_TIMEOUT_MS:-10000}
|
|
IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-[]}
|
|
IAAI_DISCOVERY_MODE: ${IAAI_DISCOVERY_MODE:-listing}
|
|
IAAI_HOURLY_MODE: ${IAAI_HOURLY_MODE:-rolling_refresh}
|
|
IAAI_HOURLY_REFRESH_BATCH_SIZE: ${IAAI_HOURLY_REFRESH_BATCH_SIZE:-500}
|
|
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
|
|
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
|
|
IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-false}
|
|
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-false}
|
|
# Первый sync после clean restart стартует сразу; следующий запуск — только через hourly guard/beat.
|
|
IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
|
|
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
|
|
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
|
|
IAAI_BROWSER_ENGINE: chromium
|
|
# Self-heal для worker.
|
|
IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
|
|
IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
|
|
IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
|
|
IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
|
|
IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
|
|
TZ: ${TZ:-UTC}
|
|
|
|
x-env-file: &env-file
|
|
- path: .env
|
|
required: false
|
|
|
|
x-app-service: &app-service
|
|
build: .
|
|
env_file: *env-file
|
|
environment: *app-env
|
|
volumes:
|
|
- ./runtime_config.json:/app/runtime_config.json:ro
|
|
|
|
x-worker-service: &worker-service
|
|
<<: *app-service
|
|
volumes:
|
|
- ./runtime_config.json:/app/runtime_config.json:ro
|
|
- tokens_data:/data
|
|
|
|
services:
|
|
# PostgreSQL.
|
|
postgres:
|
|
image: postgres:16-alpine
|
|
container_name: iaai-postgres
|
|
restart: unless-stopped
|
|
environment:
|
|
POSTGRES_USER: iaai
|
|
POSTGRES_PASSWORD: iaai
|
|
POSTGRES_DB: iaai_scraper
|
|
ports:
|
|
- "127.0.0.1:5432:5432"
|
|
volumes:
|
|
- pgdata:/var/lib/postgresql/data
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 5
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: "10m"
|
|
max-file: "5"
|
|
|
|
# Redis.
|
|
redis:
|
|
image: redis:7-alpine
|
|
container_name: iaai-redis
|
|
restart: unless-stopped
|
|
ports:
|
|
- "127.0.0.1:6379:6379"
|
|
healthcheck:
|
|
test: ["CMD", "redis-cli", "ping"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 5
|
|
command: >
|
|
redis-server
|
|
--appendonly yes
|
|
--save 60 1000
|
|
volumes:
|
|
- redisdata:/data
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: "10m"
|
|
max-file: "5"
|
|
|
|
# Миграции.
|
|
migrate:
|
|
<<: *app-service
|
|
container_name: iaai-migrate
|
|
restart: "no"
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
command: alembic upgrade head
|
|
|
|
# API.
|
|
api:
|
|
<<: *app-service
|
|
container_name: iaai-api
|
|
restart: unless-stopped
|
|
ports:
|
|
- "127.0.0.1:8000:8000"
|
|
depends_on:
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
redis:
|
|
condition: service_healthy
|
|
command: >
|
|
uvicorn iaai_scraper.api.app:app
|
|
--host 0.0.0.0 --port 8000 --workers 1
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 40s
|
|
stop_grace_period: 30s
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: "10m"
|
|
max-file: "5"
|
|
|
|
# Worker.
|
|
worker:
|
|
<<: *worker-service
|
|
restart: unless-stopped
|
|
init: true
|
|
depends_on:
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
redis:
|
|
condition: service_healthy
|
|
stop_grace_period: 60s
|
|
command: >
|
|
celery -A iaai_scraper.worker.celery_app worker
|
|
--loglevel=info
|
|
--concurrency=${CELERY_WORKER_CONCURRENCY:-4}
|
|
--pool=${CELERY_WORKER_POOL:-prefork}
|
|
--pidfile=/tmp/celery-worker.pid
|
|
-Q iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
|
|
healthcheck:
|
|
# Проверка worker.
|
|
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
|
|
interval: 60s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 90s
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: "10m"
|
|
max-file: "5"
|
|
|
|
# Beat.
|
|
beat:
|
|
<<: *worker-service
|
|
container_name: iaai-beat
|
|
restart: unless-stopped
|
|
init: true
|
|
depends_on:
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
redis:
|
|
condition: service_healthy
|
|
command: >
|
|
celery -A iaai_scraper.worker.celery_app beat
|
|
--loglevel=info
|
|
--pidfile=/tmp/celery-beat.pid
|
|
--schedule=/tmp/celerybeat-schedule
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"]
|
|
interval: 60s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 90s
|
|
logging:
|
|
driver: json-file
|
|
options:
|
|
max-size: "10m"
|
|
max-file: "5"
|
|
|
|
volumes:
|
|
pgdata:
|
|
redisdata:
|
|
tokens_data:
|