x-app-env: &app-env # Всегда используем Postgres. DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800} CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} # 0 = без лимита. CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true} CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3} DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9} DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8} DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true} DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia} DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto} DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/} DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999} DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000} DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true} # Работаем через Algolia; fallback в sitemap по умолчанию отключён. DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false} DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true} DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json} DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} DUBIZZLE_BROWSER_ENGINE: chromium # Self-heal для worker. DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true} DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900} DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} TZ: ${TZ:-UTC} x-env-file: &env-file - path: .env required: false x-app-service: &app-service build: . env_file: *env-file environment: *app-env volumes: - ./runtime_config.json:/app/runtime_config.json:ro - ./artifacts:/app/artifacts x-worker-service: &worker-service <<: *app-service volumes: - ./runtime_config.json:/app/runtime_config.json:ro - ./artifacts:/app/artifacts - tokens_data:/data services: # PostgreSQL. postgres: image: postgres:16-alpine container_name: dubizzle-postgres restart: unless-stopped environment: POSTGRES_USER: dubizzle POSTGRES_PASSWORD: dubizzle POSTGRES_DB: dubizzle_scraper ports: # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. - "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"] interval: 5s timeout: 3s retries: 5 logging: driver: json-file options: max-size: "10m" max-file: "5" # Redis. redis: image: redis:7-alpine container_name: dubizzle-redis restart: unless-stopped ports: # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. - "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s timeout: 3s retries: 5 command: > redis-server --appendonly yes --save 60 1000 volumes: - redisdata:/data logging: driver: json-file options: max-size: "10m" max-file: "5" # Миграции. migrate: <<: *app-service container_name: dubizzle-migrate restart: "no" depends_on: postgres: condition: service_healthy command: alembic upgrade head # API. api: <<: *app-service container_name: dubizzle-api restart: unless-stopped ports: # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. - "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000" depends_on: migrate: condition: service_completed_successfully redis: condition: service_healthy command: > uvicorn dubizzle_scraper.api.app:app --host 0.0.0.0 --port 8000 --workers 1 healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] interval: 30s timeout: 10s retries: 3 start_period: 40s stop_grace_period: 30s logging: driver: json-file options: max-size: "10m" max-file: "5" # Worker. worker: <<: *worker-service restart: unless-stopped init: true depends_on: migrate: condition: service_completed_successfully redis: condition: service_healthy stop_grace_period: 60s command: > celery -A dubizzle_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --pidfile=/tmp/celery-worker.pid -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} healthcheck: # Проверка worker. test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"] interval: 60s timeout: 10s retries: 3 start_period: 90s logging: driver: json-file options: max-size: "10m" max-file: "5" # Beat. beat: <<: *worker-service container_name: dubizzle-beat restart: unless-stopped init: true depends_on: migrate: condition: service_completed_successfully redis: condition: service_healthy command: > celery -A dubizzle_scraper.worker.celery_app beat --loglevel=info --pidfile=/tmp/celery-beat.pid --schedule=/tmp/celerybeat-schedule healthcheck: test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"] interval: 60s timeout: 10s retries: 3 start_period: 90s logging: driver: json-file options: max-size: "10m" max-file: "5" # Разовый прогон sync-listing через Docker CLI. sync: <<: *worker-service container_name: dubizzle-sync restart: "no" profiles: ["manual"] depends_on: migrate: condition: service_completed_successfully redis: condition: service_healthy command: > bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json" logging: driver: json-file options: max-size: "10m" max-file: "5" volumes: pgdata: redisdata: tokens_data: