281 lines
11 KiB
YAML
281 lines
11 KiB
YAML
x-app-env: &app-env
|
||
MOBILEDE_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
|
||
MOBILEDE_REDIS_URL: ${MOBILEDE_REDIS_URL:-redis://redis:6379/0}
|
||
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
|
||
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
|
||
MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS: ${MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
|
||
MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-40}
|
||
MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-80}
|
||
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
|
||
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
|
||
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
|
||
CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-6}
|
||
CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
|
||
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
|
||
MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.45}
|
||
MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-1}
|
||
MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800"
|
||
MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-}
|
||
MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-}
|
||
MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
|
||
MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true}
|
||
MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-3600}
|
||
MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10}
|
||
MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
|
||
MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
|
||
MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-5}
|
||
MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true}
|
||
MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10}
|
||
MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20}
|
||
MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50}
|
||
MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-700}
|
||
MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
|
||
# Первый проход должен идти по заранее нарезанным leaf-сегментам.
|
||
# Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально.
|
||
MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true}
|
||
MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400}
|
||
MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-320}
|
||
MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0}
|
||
MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-3}
|
||
MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN: ${MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN:-true}
|
||
MOBILEDE_FILTERED_URL_FAST_START: ${MOBILEDE_FILTERED_URL_FAST_START:-false}
|
||
MOBILEDE_REFINE_ADAPTIVE_SEGMENTS: ${MOBILEDE_REFINE_ADAPTIVE_SEGMENTS:-true}
|
||
MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS: ${MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS:-240}
|
||
MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES: ${MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES:-3}
|
||
MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8}
|
||
MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
|
||
MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true}
|
||
MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.70}
|
||
MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.90}
|
||
MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.35}
|
||
MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
|
||
MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
|
||
MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false}
|
||
MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-10}
|
||
MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED: ${MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED:-false}
|
||
MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false}
|
||
MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10}
|
||
MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true}
|
||
MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
|
||
# Автовосстановление worker.
|
||
MOBILEDE_SELF_HEAL_ENABLED: ${MOBILEDE_SELF_HEAL_ENABLED:-true}
|
||
MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
|
||
MOBILEDE_SELF_HEAL_STALL_SECONDS: ${MOBILEDE_SELF_HEAL_STALL_SECONDS:-900}
|
||
MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
|
||
MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
|
||
# При простое БД перезапускаем worker.
|
||
MOBILEDE_DB_IDLE_RESTART_SECONDS: ${MOBILEDE_DB_IDLE_RESTART_SECONDS:-3600}
|
||
TZ: ${TZ:-UTC}
|
||
|
||
x-env-file: &env-file
|
||
- path: .env
|
||
required: false
|
||
|
||
x-app-service: &app-service
|
||
build: .
|
||
env_file: *env-file
|
||
environment: *app-env
|
||
volumes:
|
||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||
|
||
x-worker-service: &worker-service
|
||
<<: *app-service
|
||
volumes:
|
||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||
- tokens_data:/data
|
||
|
||
services:
|
||
# База данных.
|
||
postgres:
|
||
image: postgres:16-alpine
|
||
restart: unless-stopped
|
||
environment:
|
||
POSTGRES_USER: ${POSTGRES_USER:-mobilede}
|
||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mobilede}
|
||
POSTGRES_DB: ${POSTGRES_DB:-mobilede_scraper}
|
||
ports:
|
||
- "127.0.0.1:${MOBILEDE_HOST_POSTGRES_PORT:-5432}:5432"
|
||
volumes:
|
||
- pgdata:/var/lib/postgresql/data
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mobilede} -d ${POSTGRES_DB:-mobilede_scraper}"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 5
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
# Очередь Redis.
|
||
redis:
|
||
image: redis:7-alpine
|
||
restart: unless-stopped
|
||
ports:
|
||
- "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379"
|
||
healthcheck:
|
||
test: ["CMD", "redis-cli", "ping"]
|
||
interval: 5s
|
||
timeout: 3s
|
||
retries: 5
|
||
command: >
|
||
redis-server
|
||
--appendonly yes
|
||
--save 60 1000
|
||
volumes:
|
||
- redisdata:/data
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
# Миграции.
|
||
migrate:
|
||
<<: *app-service
|
||
restart: "no"
|
||
depends_on:
|
||
postgres:
|
||
condition: service_healthy
|
||
command: alembic upgrade head
|
||
|
||
# API сервис.
|
||
api:
|
||
<<: *app-service
|
||
restart: unless-stopped
|
||
ports:
|
||
- "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000"
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
redis:
|
||
condition: service_healthy
|
||
command: >
|
||
uvicorn mobilede_scraper.api.app:app
|
||
--host 0.0.0.0 --port 8000 --workers 1
|
||
healthcheck:
|
||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
|
||
interval: 30s
|
||
timeout: 10s
|
||
retries: 3
|
||
start_period: 40s
|
||
stop_grace_period: 30s
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
# Worker сервис.
|
||
worker:
|
||
<<: *worker-service
|
||
restart: unless-stopped
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
redis:
|
||
condition: service_healthy
|
||
stop_grace_period: 60s
|
||
command: >
|
||
celery -A mobilede_scraper.worker.celery_app worker
|
||
--loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-8} --pool=${CELERY_WORKER_POOL:-prefork}
|
||
--pidfile=/tmp/celery-worker.pid
|
||
-Q mobilede_sync --max-tasks-per-child=5
|
||
healthcheck:
|
||
# Проверка worker.
|
||
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"]
|
||
interval: 60s
|
||
timeout: 10s
|
||
retries: 3
|
||
start_period: 90s
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
# Beat сервис.
|
||
beat:
|
||
<<: *worker-service
|
||
restart: unless-stopped
|
||
init: true
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
redis:
|
||
condition: service_healthy
|
||
command: >
|
||
celery -A mobilede_scraper.worker.celery_app beat
|
||
--loglevel=info
|
||
--pidfile=/tmp/celery-beat.pid
|
||
--schedule=/tmp/celerybeat-schedule
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"]
|
||
interval: 60s
|
||
timeout: 10s
|
||
retries: 3
|
||
start_period: 90s
|
||
logging:
|
||
driver: json-file
|
||
options:
|
||
max-size: "10m"
|
||
max-file: "5"
|
||
|
||
# Одноразовый сбор страниц поиска.
|
||
mobilede-search:
|
||
<<: *app-service
|
||
profiles: ["cli"]
|
||
restart: "no"
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
volumes:
|
||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||
- ./artifacts:/app/artifacts
|
||
command: >
|
||
mobilede search
|
||
--max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1}
|
||
--page ${MOBILEDE_SEARCH_START_PAGE:-1}
|
||
--delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
|
||
--output /app/artifacts/json/mobilede_search.json
|
||
|
||
# Одноразовый sync поиска в БД.
|
||
mobilede-sync-search:
|
||
<<: *app-service
|
||
profiles: ["cli"]
|
||
restart: "no"
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
volumes:
|
||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||
- ./artifacts:/app/artifacts
|
||
command: >
|
||
mobilede sync-search
|
||
--max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1}
|
||
--page ${MOBILEDE_SEARCH_START_PAGE:-1}
|
||
--delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
|
||
--lane ${MOBILEDE_SYNC_LANE:-mobile_de_cars}
|
||
--output /app/artifacts/json/mobilede_sync_search.json
|
||
|
||
# Одноразовый сбор карточки по ID.
|
||
mobilede-detail:
|
||
<<: *app-service
|
||
profiles: ["cli"]
|
||
restart: "no"
|
||
depends_on:
|
||
migrate:
|
||
condition: service_completed_successfully
|
||
volumes:
|
||
- ./runtime_config.json:/app/runtime_config.json:ro
|
||
- ./artifacts:/app/artifacts
|
||
command: >
|
||
mobilede detail ${MOBILEDE_LISTING_ID:-449929602}
|
||
--output /app/artifacts/json/mobilede_detail.json
|
||
|
||
volumes:
|
||
pgdata:
|
||
redisdata:
|
||
tokens_data:
|