deploy: add VPS bootstrap (WSL2+Docker) and lightweight compose overlay for 4GB RAM
This commit is contained in:
qananasikq
76
docker-compose.vps.yml
Normal file
76
docker-compose.vps.yml
Normal file
@@ -0,0 +1,76 @@
|
||||
# Overlay для слабого VPS (4 vCPU, 4 GB RAM).
|
||||
# Применять как: docker compose -f docker-compose.yml -f docker-compose.vps.yml up -d --build
|
||||
#
|
||||
# Главное:
|
||||
# - worker concurrency=1, max-tasks-per-child=3 (Chromium быстро течёт)
|
||||
# - IAAI_PARALLEL_TABS=4 (важно: каждая вкладка ~80-150 MB)
|
||||
# - IAAI_MAX_PAGES_PER_RUN=200, IAAI_MAX_VEHICLES_PER_RUN=2000 — короткие тестовые прогоны
|
||||
# - hard memory limits на каждый сервис
|
||||
|
||||
x-vps-env: &vps-env
|
||||
CELERY_WORKER_CONCURRENCY: "1"
|
||||
CELERY_WORKER_MAX_TASKS_PER_CHILD: "3"
|
||||
CELERY_WORKER_MAX_MEMORY_PER_CHILD_KB: "350000" # ~350 MB → перезапуск процесса
|
||||
IAAI_PARALLEL_TABS: "4"
|
||||
IAAI_BLOCK_RESOURCES: "true"
|
||||
IAAI_MAX_PAGES_PER_RUN: "200"
|
||||
IAAI_MAX_VEHICLES_PER_RUN: "2000"
|
||||
IAAI_LISTING_SEGMENTS: "auto"
|
||||
IAAI_HUMAN_PACE_ENABLED: "true"
|
||||
CELERY_BATCH_SIZE: "100"
|
||||
CELERY_TASK_SOFT_TIME_LIMIT: "1500" # 25 мин — короче бутстрап-сегмент
|
||||
CELERY_TASK_TIME_LIMIT: "1800" # 30 мин
|
||||
CELERY_BROKER_VISIBILITY_TIMEOUT: "3600"
|
||||
IAAI_DATABASE_POOL_SIZE: "3"
|
||||
IAAI_DATABASE_MAX_OVERFLOW: "2"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
mem_limit: 512m
|
||||
mem_reservation: 256m
|
||||
command:
|
||||
- "postgres"
|
||||
- "-c"
|
||||
- "shared_buffers=128MB"
|
||||
- "-c"
|
||||
- "effective_cache_size=384MB"
|
||||
- "-c"
|
||||
- "work_mem=8MB"
|
||||
- "-c"
|
||||
- "maintenance_work_mem=64MB"
|
||||
- "-c"
|
||||
- "max_connections=50"
|
||||
|
||||
redis:
|
||||
mem_limit: 192m
|
||||
mem_reservation: 64m
|
||||
command: >
|
||||
redis-server
|
||||
--appendonly yes
|
||||
--save 60 1000
|
||||
--maxmemory 128mb
|
||||
--maxmemory-policy allkeys-lru
|
||||
|
||||
api:
|
||||
mem_limit: 384m
|
||||
mem_reservation: 128m
|
||||
environment:
|
||||
<<: *vps-env
|
||||
|
||||
worker:
|
||||
mem_limit: 1800m
|
||||
mem_reservation: 512m
|
||||
environment:
|
||||
<<: *vps-env
|
||||
# Переопределяем, чтобы concurrency=1 и max-tasks-per-child=3 точно применились
|
||||
command: >
|
||||
celery -A iaai_scraper.worker.celery_app worker
|
||||
--loglevel=info --concurrency=1 --pool=prefork
|
||||
--pidfile=/tmp/celery-worker.pid
|
||||
-Q scraping --max-tasks-per-child=3
|
||||
|
||||
beat:
|
||||
mem_limit: 256m
|
||||
mem_reservation: 64m
|
||||
environment:
|
||||
<<: *vps-env
|
||||
121
scripts/vps_bootstrap.ps1
Normal file
121
scripts/vps_bootstrap.ps1
Normal file
@@ -0,0 +1,121 @@
|
||||
# ============================================================
|
||||
# IAAI scraper — bootstrap для Windows 10 VPS (4 vCPU / 4 GB RAM)
|
||||
# Запускать в PowerShell ОТ ИМЕНИ АДМИНИСТРАТОРА.
|
||||
#
|
||||
# Использование (через RDP):
|
||||
# 1) Откройте PowerShell как администратор
|
||||
# 2) Set-ExecutionPolicy -Scope Process Bypass -Force
|
||||
# 3) iwr -useb https://gitea.millionmiles.ru/qananasikq/iaai-parser/raw/branch/main/scripts/vps_bootstrap.ps1 | iex
|
||||
# (или скопируйте файл вручную и: .\vps_bootstrap.ps1)
|
||||
#
|
||||
# Скрипт:
|
||||
# - Включает WSL2 + ставит Ubuntu 22.04 (если ещё нет)
|
||||
# - Внутри Ubuntu ставит Docker Engine + Compose plugin + git
|
||||
# - Клонирует репо в /opt/iaai-parser
|
||||
# - Поднимает стек через docker-compose.vps.yml
|
||||
# ============================================================
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Step($msg) { Write-Host "`n=== $msg ===" -ForegroundColor Cyan }
|
||||
|
||||
if (-not ([Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) {
|
||||
throw "Run this script as Administrator (правый клик по PowerShell → 'Запуск от имени администратора')"
|
||||
}
|
||||
|
||||
# --- 1. WSL2 + Ubuntu ----------------------------------------
|
||||
Step "Checking WSL"
|
||||
$wslList = wsl --list --quiet 2>$null
|
||||
$ubuntu = $null
|
||||
if ($wslList) {
|
||||
$ubuntu = $wslList | Where-Object { $_ -match "Ubuntu" } | Select-Object -First 1
|
||||
}
|
||||
|
||||
if (-not $ubuntu) {
|
||||
Step "Installing WSL2 + Ubuntu-22.04 (this will take a few minutes; reboot may be required)"
|
||||
wsl --install -d Ubuntu-22.04 --no-launch
|
||||
Write-Host "WSL installation triggered. If a reboot is required, reboot, then re-run this script." -ForegroundColor Yellow
|
||||
Write-Host "After Ubuntu first-launches, set username 'iaai' and any password." -ForegroundColor Yellow
|
||||
Write-Host "Then re-run: .\vps_bootstrap.ps1" -ForegroundColor Yellow
|
||||
exit 0
|
||||
}
|
||||
|
||||
Write-Host "WSL distro found: $ubuntu"
|
||||
|
||||
# Чтобы WSL2 не сожрал всю RAM, ограничим до 2.5 GB
|
||||
$wslConfig = "$env:USERPROFILE\.wslconfig"
|
||||
if (-not (Test-Path $wslConfig)) {
|
||||
Step "Writing $wslConfig (memory=2500MB, processors=3, swap=1500MB)"
|
||||
@"
|
||||
[wsl2]
|
||||
memory=2500MB
|
||||
processors=3
|
||||
swap=1500MB
|
||||
localhostForwarding=true
|
||||
"@ | Out-File -FilePath $wslConfig -Encoding ASCII
|
||||
Write-Host "Restarting WSL to apply config..."
|
||||
wsl --shutdown
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
|
||||
# --- 2. Внутри WSL: docker + git + clone + up ----------------
|
||||
Step "Bootstrapping Ubuntu (docker engine, git, clone)"
|
||||
|
||||
$bash = @'
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
echo "[bootstrap] installing docker engine + compose plugin"
|
||||
sudo -n apt-get update -y
|
||||
sudo -n apt-get install -y ca-certificates curl gnupg git
|
||||
sudo -n install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -n gpg --dearmor -o /etc/apt/keyrings/docker.gpg
|
||||
sudo -n chmod a+r /etc/apt/keyrings/docker.gpg
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo -n tee /etc/apt/sources.list.d/docker.list >/dev/null
|
||||
sudo -n apt-get update -y
|
||||
sudo -n DEBIAN_FRONTEND=noninteractive apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
||||
sudo -n usermod -aG docker "$USER"
|
||||
sudo -n service docker start || true
|
||||
fi
|
||||
|
||||
# WSL2 systemd workaround: убедимся что dockerd запущен
|
||||
if ! sudo -n service docker status >/dev/null 2>&1; then
|
||||
sudo -n service docker start
|
||||
fi
|
||||
|
||||
if [ ! -d /opt/iaai-parser/.git ]; then
|
||||
sudo -n mkdir -p /opt/iaai-parser
|
||||
sudo -n chown -R "$USER:$USER" /opt/iaai-parser
|
||||
git clone https://gitea.millionmiles.ru/qananasikq/iaai-parser.git /opt/iaai-parser
|
||||
else
|
||||
cd /opt/iaai-parser && git fetch --all --prune && git reset --hard origin/main
|
||||
fi
|
||||
|
||||
cd /opt/iaai-parser
|
||||
git log -1 --oneline
|
||||
|
||||
if [ ! -f .env ]; then
|
||||
echo "[bootstrap] WARNING: .env missing — copy your .env to /opt/iaai-parser/.env, then re-run 'docker compose -f docker-compose.yml -f docker-compose.vps.yml up -d --build'"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[bootstrap] docker compose build + up (vps overlay)"
|
||||
sudo -n docker compose -f docker-compose.yml -f docker-compose.vps.yml build
|
||||
sudo -n docker compose -f docker-compose.yml -f docker-compose.vps.yml up -d
|
||||
sleep 5
|
||||
sudo -n docker compose -f docker-compose.yml -f docker-compose.vps.yml ps
|
||||
'@
|
||||
|
||||
# Заворачиваем в одну строку для wsl
|
||||
$tmpScript = "$env:TEMP\wsl_bootstrap.sh"
|
||||
$bash | Set-Content -Path $tmpScript -Encoding ASCII -NoNewline
|
||||
|
||||
# Копируем в WSL и запускаем
|
||||
wsl -d $ubuntu -- bash -c "cp /mnt/c/Users/$env:USERNAME/AppData/Local/Temp/wsl_bootstrap.sh ~/bootstrap.sh && chmod +x ~/bootstrap.sh && ~/bootstrap.sh"
|
||||
|
||||
Step "Done"
|
||||
Write-Host "Дальше:"
|
||||
Write-Host " 1) Если выведено 'WARNING: .env missing' — создайте /opt/iaai-parser/.env (см. README), затем:"
|
||||
Write-Host " wsl -d Ubuntu-22.04 -- bash -c 'cd /opt/iaai-parser && sudo docker compose -f docker-compose.yml -f docker-compose.vps.yml up -d --build'"
|
||||
Write-Host " 2) Логи: wsl -d Ubuntu-22.04 -- bash -c 'cd /opt/iaai-parser && sudo docker compose -f docker-compose.yml -f docker-compose.vps.yml logs -f --tail=80'"
|
||||
Write-Host " 3) Статус: wsl -d Ubuntu-22.04 -- bash -c 'cd /opt/iaai-parser && sudo docker compose -f docker-compose.yml -f docker-compose.vps.yml ps'"
|
||||
Reference in New Issue
Block a user