77 lines
2.3 KiB
YAML
77 lines
2.3 KiB
YAML
# Overlay для слабого VPS (4 vCPU, 4 GB RAM).
|
|
# Применять как: docker compose -f docker-compose.yml -f docker-compose.vps.yml up -d --build
|
|
#
|
|
# Главное:
|
|
# - worker concurrency=1, max-tasks-per-child=3 (Chromium быстро течёт)
|
|
# - IAAI_PARALLEL_TABS=4 (важно: каждая вкладка ~80-150 MB)
|
|
# - IAAI_MAX_PAGES_PER_RUN=200, IAAI_MAX_VEHICLES_PER_RUN=2000 — короткие тестовые прогоны
|
|
# - hard memory limits на каждый сервис
|
|
|
|
x-vps-env: &vps-env
|
|
CELERY_WORKER_CONCURRENCY: "1"
|
|
CELERY_WORKER_MAX_TASKS_PER_CHILD: "3"
|
|
CELERY_WORKER_MAX_MEMORY_PER_CHILD_KB: "350000" # ~350 MB → перезапуск процесса
|
|
IAAI_PARALLEL_TABS: "4"
|
|
IAAI_BLOCK_RESOURCES: "true"
|
|
IAAI_MAX_PAGES_PER_RUN: "200"
|
|
IAAI_MAX_VEHICLES_PER_RUN: "2000"
|
|
IAAI_LISTING_SEGMENTS: "auto"
|
|
IAAI_HUMAN_PACE_ENABLED: "true"
|
|
CELERY_BATCH_SIZE: "100"
|
|
CELERY_TASK_SOFT_TIME_LIMIT: "1500" # 25 мин — короче бутстрап-сегмент
|
|
CELERY_TASK_TIME_LIMIT: "1800" # 30 мин
|
|
CELERY_BROKER_VISIBILITY_TIMEOUT: "3600"
|
|
IAAI_DATABASE_POOL_SIZE: "3"
|
|
IAAI_DATABASE_MAX_OVERFLOW: "2"
|
|
|
|
services:
|
|
postgres:
|
|
mem_limit: 512m
|
|
mem_reservation: 256m
|
|
command:
|
|
- "postgres"
|
|
- "-c"
|
|
- "shared_buffers=128MB"
|
|
- "-c"
|
|
- "effective_cache_size=384MB"
|
|
- "-c"
|
|
- "work_mem=8MB"
|
|
- "-c"
|
|
- "maintenance_work_mem=64MB"
|
|
- "-c"
|
|
- "max_connections=50"
|
|
|
|
redis:
|
|
mem_limit: 192m
|
|
mem_reservation: 64m
|
|
command: >
|
|
redis-server
|
|
--appendonly yes
|
|
--save 60 1000
|
|
--maxmemory 128mb
|
|
--maxmemory-policy allkeys-lru
|
|
|
|
api:
|
|
mem_limit: 384m
|
|
mem_reservation: 128m
|
|
environment:
|
|
<<: *vps-env
|
|
|
|
worker:
|
|
mem_limit: 1800m
|
|
mem_reservation: 512m
|
|
environment:
|
|
<<: *vps-env
|
|
# Переопределяем, чтобы concurrency=1 и max-tasks-per-child=3 точно применились
|
|
command: >
|
|
celery -A iaai_scraper.worker.celery_app worker
|
|
--loglevel=info --concurrency=1 --pool=prefork
|
|
--pidfile=/tmp/celery-worker.pid
|
|
-Q scraping --max-tasks-per-child=3
|
|
|
|
beat:
|
|
mem_limit: 256m
|
|
mem_reservation: 64m
|
|
environment:
|
|
<<: *vps-env
|