From eb9fb48ea0a425835ef9721d73a515786a991dfd Mon Sep 17 00:00:00 2001 From: qananasikq Date: Thu, 9 Apr 2026 20:34:34 +0300 Subject: [PATCH] improve docker compose setup --- .env.example | 2 + alembic/env.py | 6 +- alembic/versions/002_add_indexes.py | 47 +++++++++ docker-compose.yml | 155 ++++++++++++++++++++-------- entrypoint.sh | 94 ++++++++++------- 5 files changed, 223 insertions(+), 81 deletions(-) create mode 100644 alembic/versions/002_add_indexes.py diff --git a/.env.example b/.env.example index 3b193ec..bde417e 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,8 @@ IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0 # Sync settings IAAI_SYNC_ONLY_NEW=true +IAAI_TOKENS_FILE=/data/tokens.json +IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json # Retry / backoff IAAI_RETRY_DELAY_SECONDS=2.5 diff --git a/alembic/env.py b/alembic/env.py index 03ffa6e..f6f4ab0 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,4 +1,4 @@ -"""Alembic env.py — подключение к БД через Settings.""" +# Alembic env.py — подключение к БД через Settings. import os import sys @@ -27,7 +27,7 @@ target_metadata = Base.metadata def run_migrations_offline() -> None: - """Run migrations in 'offline' mode.""" + # Run migrations in 'offline' mode. url = config.get_main_option("sqlalchemy.url") context.configure( url=url, @@ -40,7 +40,7 @@ def run_migrations_offline() -> None: def run_migrations_online() -> None: - """Run migrations in 'online' mode.""" + # Run migrations in 'online' mode. connectable = engine_from_config( config.get_section(config.config_ini_section, {}), prefix="sqlalchemy.", diff --git a/alembic/versions/002_add_indexes.py b/alembic/versions/002_add_indexes.py new file mode 100644 index 0000000..284f3d8 --- /dev/null +++ b/alembic/versions/002_add_indexes.py @@ -0,0 +1,47 @@ +"""Add performance indexes for growing database + +Revision ID: 002_add_indexes +Revises: 001_initial +Create Date: 2026-04-09 +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "002_add_indexes" +down_revision: Union[str, None] = "001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # cars: ускорение фильтрации по бренду в API и статистике + op.create_index("ix_cars_brand", "cars", ["brand"]) + + # cars: составной индекс бренд+модель для комбинированных фильтров + op.create_index("ix_cars_brand_model", "cars", ["brand", "model"]) + + # cars: ускорение фильтрации по году (year_min/year_max) + op.create_index("ix_cars_year", "cars", ["year"]) + + # cars: ускорение фильтрации по статусу продажи + op.create_index("ix_cars_is_sold", "cars", ["is_sold"]) + + # cars: ускорение сортировки ORDER BY last_seen_at DESC (пагинация) + op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"]) + + # images: ускорение JOIN/DELETE по car_id (критично при upsert) + op.create_index("ix_images_car_id", "images", ["car_id"]) + + # sync_runs: ускорение поиска stale runs по статусу + op.create_index("ix_sync_runs_status", "sync_runs", ["status"]) + + +def downgrade() -> None: + op.drop_index("ix_sync_runs_status", table_name="sync_runs") + op.drop_index("ix_images_car_id", table_name="images") + op.drop_index("ix_cars_last_seen_at", table_name="cars") + op.drop_index("ix_cars_is_sold", table_name="cars") + op.drop_index("ix_cars_year", table_name="cars") + op.drop_index("ix_cars_brand_model", table_name="cars") + op.drop_index("ix_cars_brand", table_name="cars") diff --git a/docker-compose.yml b/docker-compose.yml index 750c37f..a62605b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,36 @@ +x-app-env: &app-env + IAAI_DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper} + IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} + CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} + IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800} + CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} + CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} + CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200} + CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-20} + IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json} + IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} + TZ: ${TZ:-UTC} + +x-env-file: &env-file + - path: .env + required: false + +x-app-service: &app-service + build: . + env_file: *env-file + environment: *app-env + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + +x-worker-service: &worker-service + <<: *app-service + volumes: + - ./runtime_config.json:/app/runtime_config.json:ro + - tokens_data:/data + services: - # ─── PostgreSQL ─────────────────────────────────────────── + # PostgreSQL postgres: image: postgres:16-alpine container_name: iaai-postgres @@ -17,8 +48,13 @@ services: interval: 5s timeout: 3s retries: 5 + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" - # ─── Redis (Celery broker) ──────────────────────────────── + # Redis (Celery broker) redis: image: redis:7-alpine container_name: iaai-redis @@ -30,76 +66,109 @@ services: interval: 5s timeout: 3s retries: 5 + command: > + redis-server + --appendonly yes + --save 60 1000 + volumes: + - redisdata:/data + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" - # ─── FastAPI ────────────────────────────────────────────── - api: - build: . - container_name: iaai-api - restart: unless-stopped - env_file: - - path: .env - required: false - environment: - IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper - IAAI_REDIS_URL: redis://redis:6379/0 - CELERY_BROKER_URL: redis://redis:6379/0 - CELERY_RESULT_BACKEND: redis://redis:6379/0 - ports: - - "8000:8000" + # DB migrations + migrate: + <<: *app-service + container_name: iaai-migrate + restart: "no" depends_on: postgres: condition: service_healthy + command: alembic upgrade head + + # FastAPI + api: + <<: *app-service + container_name: iaai-api + restart: unless-stopped + ports: + - "8000:8000" + depends_on: + migrate: + condition: service_completed_successfully redis: condition: service_healthy command: > uvicorn iaai_scraper.api.app:app - --host 0.0.0.0 --port 8000 --workers 2 + --host 0.0.0.0 --port 8000 --workers 1 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + stop_grace_period: 30s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" - # ─── Celery Worker ──────────────────────────────────────── + # Celery Worker worker: - build: . + <<: *worker-service container_name: iaai-worker restart: unless-stopped - env_file: - - path: .env - required: false - environment: - IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper - IAAI_REDIS_URL: redis://redis:6379/0 - CELERY_BROKER_URL: redis://redis:6379/0 - CELERY_RESULT_BACKEND: redis://redis:6379/0 depends_on: - postgres: - condition: service_healthy + migrate: + condition: service_completed_successfully redis: condition: service_healthy stop_grace_period: 60s command: > celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=prefork - -Q scraping --without-heartbeat + -Q scraping --max-tasks-per-child=20 + healthcheck: + test: ["CMD", "celery", "-A", "iaai_scraper.worker.celery_app", "inspect", "ping", "-d", "celery@$$HOSTNAME"] + interval: 60s + timeout: 20s + retries: 3 + start_period: 40s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" - # ─── Celery Beat (периодический планировщик) ────────────── + # Celery Beat (периодический планировщик) beat: - build: . + <<: *worker-service container_name: iaai-beat restart: unless-stopped - env_file: - - path: .env - required: false - environment: - IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper - IAAI_REDIS_URL: redis://redis:6379/0 - CELERY_BROKER_URL: redis://redis:6379/0 - CELERY_RESULT_BACKEND: redis://redis:6379/0 depends_on: - postgres: - condition: service_healthy + migrate: + condition: service_completed_successfully redis: condition: service_healthy command: > celery -A iaai_scraper.worker.celery_app beat --loglevel=info + healthcheck: + test: ["CMD", "python", "-c", "import pathlib,sys; p=pathlib.Path('/tmp/celerybeat-schedule'); sys.exit(0 if p.exists() else 1)"] + interval: 60s + timeout: 20s + retries: 3 + start_period: 60s + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" volumes: pgdata: + redisdata: + tokens_data: diff --git a/entrypoint.sh b/entrypoint.sh index 869b0cc..92e1676 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,57 +1,81 @@ #!/bin/bash -set -e +set -euo pipefail -# Start HTTP→SOCKS5 proxy bridge if SOCKS5 upstream is configured -if [ -n "$SOCKS5_PROXY_HOST" ]; then - echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 $SOCKS5_PROXY_HOST:${SOCKS5_PROXY_PORT:-1002})..." - if [ -z "$IAAI_PROXY_SERVER" ]; then +is_worker_command() { + local joined="$*" + case "$joined" in + *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*) + return 0 + ;; + esac + return 1 +} + +is_scrape_cli_command() { + local joined="$*" + case "$joined" in + *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*) + return 0 + ;; + esac + return 1 +} + +needs_browser_runtime() { + is_worker_command "$@" || is_scrape_cli_command "$@" +} + +start_proxy_bridge_if_needed() { + if ! needs_browser_runtime "$@"; then + return 0 + fi + + if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then + return 0 + fi + + echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..." + + if [ -z "${IAAI_PROXY_SERVER:-}" ]; then export IAAI_PROXY_SERVER="http://127.0.0.1:8899" - elif [ "$IAAI_PROXY_SERVER" = "http://localhost:8899" ]; then + elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then export IAAI_PROXY_SERVER="http://127.0.0.1:8899" fi - echo "[entrypoint] Using browser proxy: $IAAI_PROXY_SERVER" + + echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}" python -m iaai_scraper.proxy_bridge & BRIDGE_PID=$! sleep 1 - if ! kill -0 $BRIDGE_PID 2>/dev/null; then + + if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start" exit 1 fi - echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)" -fi -# Xvfb needed only for worker (Playwright) — not for API or beat -NEEDS_XVFB=false -case "$1" in - celery*|python*main*) - NEEDS_XVFB=true - ;; - *) - # Check if any arg contains "worker" - for arg in "$@"; do - case "$arg" in - *worker*) NEEDS_XVFB=true; break ;; - esac - done - ;; -esac + echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})" +} + +start_xvfb_if_needed() { + if ! needs_browser_runtime "$@"; then + return 0 + fi -if [ "$NEEDS_XVFB" = "true" ]; then # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display export DISPLAY=:99 export IAAI_HEADLESS=false + + if [ -S /tmp/.X11-unix/X99 ] || [ -f /tmp/.X99-lock ]; then + echo "[entrypoint] Reusing existing Xvfb on DISPLAY=${DISPLAY}" + return 0 + fi + Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp & XVFB_PID=$! sleep 0.5 - echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)" -fi + echo "[entrypoint] Xvfb started (PID ${XVFB_PID}, DISPLAY=${DISPLAY})" +} -# Run Alembic migrations (only for API service, skip for worker/beat) -case "$1" in - uvicorn*) - echo "[entrypoint] Running Alembic migrations..." - alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..." - ;; -esac +start_proxy_bridge_if_needed "$@" +start_xvfb_if_needed "$@" exec "$@"