dubizzle project
This commit is contained in:
qananasikq
228
docker-compose.yml
Normal file
228
docker-compose.yml
Normal file
@@ -0,0 +1,228 @@
|
||||
x-app-env: &app-env
|
||||
# Всегда используем Postgres.
|
||||
DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
|
||||
DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0}
|
||||
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
|
||||
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
|
||||
DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
|
||||
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
|
||||
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
|
||||
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
|
||||
# 0 = без лимита.
|
||||
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
|
||||
CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true}
|
||||
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
|
||||
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
|
||||
DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3}
|
||||
DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9}
|
||||
DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8}
|
||||
DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true}
|
||||
DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia}
|
||||
DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto}
|
||||
DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/}
|
||||
DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999}
|
||||
DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000}
|
||||
DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true}
|
||||
# Работаем через Algolia; fallback в sitemap по умолчанию отключён.
|
||||
DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false}
|
||||
DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true}
|
||||
DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json}
|
||||
DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
|
||||
DUBIZZLE_BROWSER_ENGINE: chromium
|
||||
# Self-heal для worker.
|
||||
DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true}
|
||||
DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
|
||||
DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900}
|
||||
DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
|
||||
DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
|
||||
TZ: ${TZ:-UTC}
|
||||
|
||||
x-env-file: &env-file
|
||||
- path: .env
|
||||
required: false
|
||||
|
||||
x-app-service: &app-service
|
||||
build: .
|
||||
env_file: *env-file
|
||||
environment: *app-env
|
||||
volumes:
|
||||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||||
- ./artifacts:/app/artifacts
|
||||
|
||||
x-worker-service: &worker-service
|
||||
<<: *app-service
|
||||
volumes:
|
||||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||||
- ./artifacts:/app/artifacts
|
||||
- tokens_data:/data
|
||||
|
||||
services:
|
||||
# PostgreSQL.
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
container_name: dubizzle-postgres
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: dubizzle
|
||||
POSTGRES_PASSWORD: dubizzle
|
||||
POSTGRES_DB: dubizzle_scraper
|
||||
ports:
|
||||
# Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
|
||||
- "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432"
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# Redis.
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: dubizzle-redis
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
# Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
|
||||
- "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379"
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
command: >
|
||||
redis-server
|
||||
--appendonly yes
|
||||
--save 60 1000
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# Миграции.
|
||||
migrate:
|
||||
<<: *app-service
|
||||
container_name: dubizzle-migrate
|
||||
restart: "no"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
command: alembic upgrade head
|
||||
|
||||
# API.
|
||||
api:
|
||||
<<: *app-service
|
||||
container_name: dubizzle-api
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
# Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
|
||||
- "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000"
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
command: >
|
||||
uvicorn dubizzle_scraper.api.app:app
|
||||
--host 0.0.0.0 --port 8000 --workers 1
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
stop_grace_period: 30s
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# Worker.
|
||||
worker:
|
||||
<<: *worker-service
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
stop_grace_period: 60s
|
||||
command: >
|
||||
celery -A dubizzle_scraper.worker.celery_app worker
|
||||
--loglevel=info --concurrency=1 --pool=solo
|
||||
--pidfile=/tmp/celery-worker.pid
|
||||
-Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
|
||||
healthcheck:
|
||||
# Проверка worker.
|
||||
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"]
|
||||
interval: 60s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 90s
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# Beat.
|
||||
beat:
|
||||
<<: *worker-service
|
||||
container_name: dubizzle-beat
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
command: >
|
||||
celery -A dubizzle_scraper.worker.celery_app beat
|
||||
--loglevel=info
|
||||
--pidfile=/tmp/celery-beat.pid
|
||||
--schedule=/tmp/celerybeat-schedule
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"]
|
||||
interval: 60s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 90s
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
# Разовый прогон sync-listing через Docker CLI.
|
||||
sync:
|
||||
<<: *worker-service
|
||||
container_name: dubizzle-sync
|
||||
restart: "no"
|
||||
profiles: ["manual"]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
command: >
|
||||
bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json"
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "5"
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
tokens_data:
|
||||
Reference in New Issue
Block a user