improve docker compose setup

2026-04-09 20:34:34 +03:00
parent 042ffdcfbb
commit eb9fb48ea0
5 changed files with 223 additions and 81 deletions
--- a/.env.example
+++ b/.env.example
@@ -32,6 +32,8 @@ IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0

 # Sync settings
 IAAI_SYNC_ONLY_NEW=true
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json

 # Retry / backoff
 IAAI_RETRY_DELAY_SECONDS=2.5
--- a/alembic/env.py
+++ b/alembic/env.py
@@ -1,4 +1,4 @@
-"""Alembic env.py — подключение к БД через Settings."""
+# Alembic env.py — подключение к БД через Settings.

 import os
 import sys
@@ -27,7 +27,7 @@ target_metadata = Base.metadata


 def run_migrations_offline() -> None:
-    """Run migrations in 'offline' mode."""
+    # Run migrations in 'offline' mode.
    url = config.get_main_option("sqlalchemy.url")
    context.configure(
        url=url,
@@ -40,7 +40,7 @@ def run_migrations_offline() -> None:


 def run_migrations_online() -> None:
-    """Run migrations in 'online' mode."""
+    # Run migrations in 'online' mode.
    connectable = engine_from_config(
        config.get_section(config.config_ini_section, {}),
        prefix="sqlalchemy.",
--- a/alembic/versions/002_add_indexes.py
+++ b/alembic/versions/002_add_indexes.py
@@ -0,0 +1,47 @@
+"""Add performance indexes for growing database
+
+Revision ID: 002_add_indexes
+Revises: 001_initial
+Create Date: 2026-04-09
+"""
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "002_add_indexes"
+down_revision: Union[str, None] = "001_initial"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # cars: ускорение фильтрации по бренду в API и статистике
+    op.create_index("ix_cars_brand", "cars", ["brand"])
+
+    # cars: составной индекс бренд+модель для комбинированных фильтров
+    op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
+
+    # cars: ускорение фильтрации по году (year_min/year_max)
+    op.create_index("ix_cars_year", "cars", ["year"])
+
+    # cars: ускорение фильтрации по статусу продажи
+    op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
+
+    # cars: ускорение сортировки ORDER BY last_seen_at DESC (пагинация)
+    op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
+
+    # images: ускорение JOIN/DELETE по car_id (критично при upsert)
+    op.create_index("ix_images_car_id", "images", ["car_id"])
+
+    # sync_runs: ускорение поиска stale runs по статусу
+    op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_sync_runs_status", table_name="sync_runs")
+    op.drop_index("ix_images_car_id", table_name="images")
+    op.drop_index("ix_cars_last_seen_at", table_name="cars")
+    op.drop_index("ix_cars_is_sold", table_name="cars")
+    op.drop_index("ix_cars_year", table_name="cars")
+    op.drop_index("ix_cars_brand_model", table_name="cars")
+    op.drop_index("ix_cars_brand", table_name="cars")
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,36 @@
+x-app-env: &app-env
+  IAAI_DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper}
+  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
+  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
+  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-20}
+  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  TZ: ${TZ:-UTC}
+
+x-env-file: &env-file
+  - path: .env
+    required: false
+
+x-app-service: &app-service
+  build: .
+  env_file: *env-file
+  environment: *app-env
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+
+x-worker-service: &worker-service
+  <<: *app-service
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+    - tokens_data:/data
+
 services:
-  # ─── PostgreSQL ───────────────────────────────────────────
+  # PostgreSQL
  postgres:
    image: postgres:16-alpine
    container_name: iaai-postgres
@@ -17,8 +48,13 @@ services:
      interval: 5s
      timeout: 3s
      retries: 5
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

-  # ─── Redis (Celery broker) ────────────────────────────────
+  # Redis (Celery broker)
  redis:
    image: redis:7-alpine
    container_name: iaai-redis
@@ -30,76 +66,109 @@ services:
      interval: 5s
      timeout: 3s
      retries: 5
+    command: >
+      redis-server
+      --appendonly yes
+      --save 60 1000
+    volumes:
+      - redisdata:/data
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

-  # ─── FastAPI ──────────────────────────────────────────────
-  api:
-    build: .
-    container_name: iaai-api
-    restart: unless-stopped
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
-    ports:
-      - "8000:8000"
+  # DB migrations
+  migrate:
+    <<: *app-service
+    container_name: iaai-migrate
+    restart: "no"
    depends_on:
      postgres:
        condition: service_healthy
+    command: alembic upgrade head
+
+  # FastAPI
+  api:
+    <<: *app-service
+    container_name: iaai-api
+    restart: unless-stopped
+    ports:
+      - "8000:8000"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
      redis:
        condition: service_healthy
    command: >
      uvicorn iaai_scraper.api.app:app
-      --host 0.0.0.0 --port 8000 --workers 2
+      --host 0.0.0.0 --port 8000 --workers 1
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    stop_grace_period: 30s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

-  # ─── Celery Worker ────────────────────────────────────────
+  # Celery Worker
  worker:
-    build: .
+    <<: *worker-service
    container_name: iaai-worker
    restart: unless-stopped
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
    depends_on:
-      postgres:
-        condition: service_healthy
+      migrate:
+        condition: service_completed_successfully
      redis:
        condition: service_healthy
    stop_grace_period: 60s
    command: >
      celery -A iaai_scraper.worker.celery_app worker
      --loglevel=info --concurrency=1 --pool=prefork
-      -Q scraping --without-heartbeat
+      -Q scraping --max-tasks-per-child=20
+    healthcheck:
+      test: ["CMD", "celery", "-A", "iaai_scraper.worker.celery_app", "inspect", "ping", "-d", "celery@$$HOSTNAME"]
+      interval: 60s
+      timeout: 20s
+      retries: 3
+      start_period: 40s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

-  # ─── Celery Beat (периодический планировщик) ──────────────
+  # Celery Beat (периодический планировщик)
  beat:
-    build: .
+    <<: *worker-service
    container_name: iaai-beat
    restart: unless-stopped
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
    depends_on:
-      postgres:
-        condition: service_healthy
+      migrate:
+        condition: service_completed_successfully
      redis:
        condition: service_healthy
    command: >
      celery -A iaai_scraper.worker.celery_app beat
      --loglevel=info
+    healthcheck:
+      test: ["CMD", "python", "-c", "import pathlib,sys; p=pathlib.Path('/tmp/celerybeat-schedule'); sys.exit(0 if p.exists() else 1)"]
+      interval: 60s
+      timeout: 20s
+      retries: 3
+      start_period: 60s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"

 volumes:
  pgdata:
+  redisdata:
+  tokens_data:
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -1,57 +1,81 @@
 #!/bin/bash
-set -e
+set -euo pipefail

-# Start HTTP→SOCKS5 proxy bridge if SOCKS5 upstream is configured
-if [ -n "$SOCKS5_PROXY_HOST" ]; then
-  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 $SOCKS5_PROXY_HOST:${SOCKS5_PROXY_PORT:-1002})..."
-  if [ -z "$IAAI_PROXY_SERVER" ]; then
+is_worker_command() {
+  local joined="$*"
+  case "$joined" in
+    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+is_scrape_cli_command() {
+  local joined="$*"
+  case "$joined" in
+    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+needs_browser_runtime() {
+  is_worker_command "$@" || is_scrape_cli_command "$@"
+}
+
+start_proxy_bridge_if_needed() {
+  if ! needs_browser_runtime "$@"; then
+    return 0
+  fi
+
+  if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
+    return 0
+  fi
+
+  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
+
+  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "$IAAI_PROXY_SERVER" = "http://localhost:8899" ]; then
+  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
  fi
-  echo "[entrypoint] Using browser proxy: $IAAI_PROXY_SERVER"
+
+  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
  python -m iaai_scraper.proxy_bridge &
  BRIDGE_PID=$!
  sleep 1
-  if ! kill -0 $BRIDGE_PID 2>/dev/null; then
+
+  if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
    exit 1
  fi
-  echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)"
-fi

-# Xvfb needed only for worker (Playwright) — not for API or beat
-NEEDS_XVFB=false
-case "$1" in
-  celery*|python*main*)
-    NEEDS_XVFB=true
-    ;;
-  *)
-    # Check if any arg contains "worker"
-    for arg in "$@"; do
-      case "$arg" in
-        *worker*) NEEDS_XVFB=true; break ;;
-      esac
-    done
-    ;;
-esac
+  echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
+}
+
+start_xvfb_if_needed() {
+  if ! needs_browser_runtime "$@"; then
+    return 0
+  fi

-if [ "$NEEDS_XVFB" = "true" ]; then
  # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display
  export DISPLAY=:99
  export IAAI_HEADLESS=false
+
+  if [ -S /tmp/.X11-unix/X99 ] || [ -f /tmp/.X99-lock ]; then
+    echo "[entrypoint] Reusing existing Xvfb on DISPLAY=${DISPLAY}"
+    return 0
+  fi
+
  Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
  XVFB_PID=$!
  sleep 0.5
-  echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)"
-fi
+  echo "[entrypoint] Xvfb started (PID ${XVFB_PID}, DISPLAY=${DISPLAY})"
+}

-# Run Alembic migrations (only for API service, skip for worker/beat)
-case "$1" in
-  uvicorn*)
-    echo "[entrypoint] Running Alembic migrations..."
-    alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..."
-    ;;
-esac
+start_proxy_bridge_if_needed "$@"
+start_xvfb_if_needed "$@"

 exec "$@"