diff --git a/docker-compose.vps.yml b/docker-compose.vps.yml new file mode 100644 index 0000000..ecf5694 --- /dev/null +++ b/docker-compose.vps.yml @@ -0,0 +1,76 @@ +# Overlay для слабого VPS (4 vCPU, 4 GB RAM). +# Применять как: docker compose -f docker-compose.yml -f docker-compose.vps.yml up -d --build +# +# Главное: +# - worker concurrency=1, max-tasks-per-child=3 (Chromium быстро течёт) +# - IAAI_PARALLEL_TABS=4 (важно: каждая вкладка ~80-150 MB) +# - IAAI_MAX_PAGES_PER_RUN=200, IAAI_MAX_VEHICLES_PER_RUN=2000 — короткие тестовые прогоны +# - hard memory limits на каждый сервис + +x-vps-env: &vps-env + CELERY_WORKER_CONCURRENCY: "1" + CELERY_WORKER_MAX_TASKS_PER_CHILD: "3" + CELERY_WORKER_MAX_MEMORY_PER_CHILD_KB: "350000" # ~350 MB → перезапуск процесса + IAAI_PARALLEL_TABS: "4" + IAAI_BLOCK_RESOURCES: "true" + IAAI_MAX_PAGES_PER_RUN: "200" + IAAI_MAX_VEHICLES_PER_RUN: "2000" + IAAI_LISTING_SEGMENTS: "auto" + IAAI_HUMAN_PACE_ENABLED: "true" + CELERY_BATCH_SIZE: "100" + CELERY_TASK_SOFT_TIME_LIMIT: "1500" # 25 мин — короче бутстрап-сегмент + CELERY_TASK_TIME_LIMIT: "1800" # 30 мин + CELERY_BROKER_VISIBILITY_TIMEOUT: "3600" + IAAI_DATABASE_POOL_SIZE: "3" + IAAI_DATABASE_MAX_OVERFLOW: "2" + +services: + postgres: + mem_limit: 512m + mem_reservation: 256m + command: + - "postgres" + - "-c" + - "shared_buffers=128MB" + - "-c" + - "effective_cache_size=384MB" + - "-c" + - "work_mem=8MB" + - "-c" + - "maintenance_work_mem=64MB" + - "-c" + - "max_connections=50" + + redis: + mem_limit: 192m + mem_reservation: 64m + command: > + redis-server + --appendonly yes + --save 60 1000 + --maxmemory 128mb + --maxmemory-policy allkeys-lru + + api: + mem_limit: 384m + mem_reservation: 128m + environment: + <<: *vps-env + + worker: + mem_limit: 1800m + mem_reservation: 512m + environment: + <<: *vps-env + # Переопределяем, чтобы concurrency=1 и max-tasks-per-child=3 точно применились + command: > + celery -A iaai_scraper.worker.celery_app worker + --loglevel=info --concurrency=1 --pool=prefork + --pidfile=/tmp/celery-worker.pid + -Q scraping --max-tasks-per-child=3 + + beat: + mem_limit: 256m + mem_reservation: 64m + environment: + <<: *vps-env