improve docker compose setup

.env.example

@@ -32,6 +32,8 @@ IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
 
 # Sync settings
 IAAI_SYNC_ONLY_NEW=true
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 
 # Retry / backoff
 IAAI_RETRY_DELAY_SECONDS=2.5

alembic/env.py

@@ -1,4 +1,4 @@
-"""Alembic env.py — подключение к БД через Settings."""
+# Alembic env.py — подключение к БД через Settings.
 
 import os
 import sys
@@ -27,7 +27,7 @@ target_metadata = Base.metadata
 
 
 def run_migrations_offline() -> None:
-    """Run migrations in 'offline' mode."""
+    # Run migrations in 'offline' mode.
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
         url=url,
@@ -40,7 +40,7 @@ def run_migrations_offline() -> None:
 
 
 def run_migrations_online() -> None:
-    """Run migrations in 'online' mode."""
+    # Run migrations in 'online' mode.
     connectable = engine_from_config(
         config.get_section(config.config_ini_section, {}),
         prefix="sqlalchemy.",

alembic/versions/002_add_indexes.py

@@ -0,0 +1,47 @@
+"""Add performance indexes for growing database
+
+Revision ID: 002_add_indexes
+Revises: 001_initial
+Create Date: 2026-04-09
+"""
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "002_add_indexes"
+down_revision: Union[str, None] = "001_initial"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # cars: ускорение фильтрации по бренду в API и статистике
+    op.create_index("ix_cars_brand", "cars", ["brand"])
+
+    # cars: составной индекс бренд+модель для комбинированных фильтров
+    op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
+
+    # cars: ускорение фильтрации по году (year_min/year_max)
+    op.create_index("ix_cars_year", "cars", ["year"])
+
+    # cars: ускорение фильтрации по статусу продажи
+    op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
+
+    # cars: ускорение сортировки ORDER BY last_seen_at DESC (пагинация)
+    op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
+
+    # images: ускорение JOIN/DELETE по car_id (критично при upsert)
+    op.create_index("ix_images_car_id", "images", ["car_id"])
+
+    # sync_runs: ускорение поиска stale runs по статусу
+    op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
+
+
+def downgrade() -> None:
+    op.drop_index("ix_sync_runs_status", table_name="sync_runs")
+    op.drop_index("ix_images_car_id", table_name="images")
+    op.drop_index("ix_cars_last_seen_at", table_name="cars")
+    op.drop_index("ix_cars_is_sold", table_name="cars")
+    op.drop_index("ix_cars_year", table_name="cars")
+    op.drop_index("ix_cars_brand_model", table_name="cars")
+    op.drop_index("ix_cars_brand", table_name="cars")

docker-compose.yml

@@ -1,5 +1,36 @@
+x-app-env: &app-env
+  IAAI_DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper}
+  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
+  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
+  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-20}
+  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  TZ: ${TZ:-UTC}
+
+x-env-file: &env-file
+  - path: .env
+    required: false
+
+x-app-service: &app-service
+  build: .
+  env_file: *env-file
+  environment: *app-env
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+
+x-worker-service: &worker-service
+  <<: *app-service
+  volumes:
+    - ./runtime_config.json:/app/runtime_config.json:ro
+    - tokens_data:/data
+
 services:
-  # ─── PostgreSQL ───────────────────────────────────────────
+  # PostgreSQL
   postgres:
     image: postgres:16-alpine
     container_name: iaai-postgres
@@ -17,8 +48,13 @@ services:
       interval: 5s
       timeout: 3s
       retries: 5
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
-  # ─── Redis (Celery broker) ────────────────────────────────
+  # Redis (Celery broker)
   redis:
     image: redis:7-alpine
     container_name: iaai-redis
@@ -30,76 +66,109 @@ services:
       interval: 5s
       timeout: 3s
       retries: 5
+    command: >
+      redis-server
+      --appendonly yes
+      --save 60 1000
+    volumes:
+      - redisdata:/data
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
-  # ─── FastAPI ──────────────────────────────────────────────
+  # DB migrations
-  api:
+  migrate:
-    build: .
+    <<: *app-service
-    container_name: iaai-api
+    container_name: iaai-migrate
-    restart: unless-stopped
+    restart: "no"
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
-    ports:
-      - "8000:8000"
     depends_on:
       postgres:
         condition: service_healthy
+    command: alembic upgrade head
+
+  # FastAPI
+  api:
+    <<: *app-service
+    container_name: iaai-api
+    restart: unless-stopped
+    ports:
+      - "8000:8000"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
       uvicorn iaai_scraper.api.app:app
-      --host 0.0.0.0 --port 8000 --workers 2
+      --host 0.0.0.0 --port 8000 --workers 1
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    stop_grace_period: 30s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
-  # ─── Celery Worker ────────────────────────────────────────
+  # Celery Worker
   worker:
-    build: .
+    <<: *worker-service
     container_name: iaai-worker
     restart: unless-stopped
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
     depends_on:
-      postgres:
+      migrate:
-        condition: service_healthy
+        condition: service_completed_successfully
       redis:
         condition: service_healthy
     stop_grace_period: 60s
     command: >
       celery -A iaai_scraper.worker.celery_app worker
       --loglevel=info --concurrency=1 --pool=prefork
-      -Q scraping --without-heartbeat
+      -Q scraping --max-tasks-per-child=20
+    healthcheck:
+      test: ["CMD", "celery", "-A", "iaai_scraper.worker.celery_app", "inspect", "ping", "-d", "celery@$$HOSTNAME"]
+      interval: 60s
+      timeout: 20s
+      retries: 3
+      start_period: 40s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
-  # ─── Celery Beat (периодический планировщик) ──────────────
+  # Celery Beat (периодический планировщик)
   beat:
-    build: .
+    <<: *worker-service
     container_name: iaai-beat
     restart: unless-stopped
-    env_file:
-      - path: .env
-        required: false
-    environment:
-      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-      IAAI_REDIS_URL: redis://redis:6379/0
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
     depends_on:
-      postgres:
+      migrate:
-        condition: service_healthy
+        condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
       celery -A iaai_scraper.worker.celery_app beat
       --loglevel=info
+    healthcheck:
+      test: ["CMD", "python", "-c", "import pathlib,sys; p=pathlib.Path('/tmp/celerybeat-schedule'); sys.exit(0 if p.exists() else 1)"]
+      interval: 60s
+      timeout: 20s
+      retries: 3
+      start_period: 60s
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
 
 volumes:
   pgdata:
+  redisdata:
+  tokens_data:

entrypoint.sh

@@ -1,57 +1,81 @@
 #!/bin/bash
-set -e
+set -euo pipefail
 
-# Start HTTP→SOCKS5 proxy bridge if SOCKS5 upstream is configured
+is_worker_command() {
-if [ -n "$SOCKS5_PROXY_HOST" ]; then
+  local joined="$*"
-  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 $SOCKS5_PROXY_HOST:${SOCKS5_PROXY_PORT:-1002})..."
+  case "$joined" in
-  if [ -z "$IAAI_PROXY_SERVER" ]; then
+    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+is_scrape_cli_command() {
+  local joined="$*"
+  case "$joined" in
+    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
+      return 0
+      ;;
+  esac
+  return 1
+}
+
+needs_browser_runtime() {
+  is_worker_command "$@" || is_scrape_cli_command "$@"
+}
+
+start_proxy_bridge_if_needed() {
+  if ! needs_browser_runtime "$@"; then
+    return 0
+  fi
+
+  if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
+    return 0
+  fi
+
+  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
+
+  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
     export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "$IAAI_PROXY_SERVER" = "http://localhost:8899" ]; then
+  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
     export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
   fi
-  echo "[entrypoint] Using browser proxy: $IAAI_PROXY_SERVER"
+
+  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
   python -m iaai_scraper.proxy_bridge &
   BRIDGE_PID=$!
   sleep 1
-  if ! kill -0 $BRIDGE_PID 2>/dev/null; then
+
+  if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
     echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
     exit 1
   fi
-  echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)"
+
+  echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
+}
+
+start_xvfb_if_needed() {
+  if ! needs_browser_runtime "$@"; then
+    return 0
   fi
 
-# Xvfb needed only for worker (Playwright) — not for API or beat
-NEEDS_XVFB=false
-case "$1" in
-  celery*|python*main*)
-    NEEDS_XVFB=true
-    ;;
-  *)
-    # Check if any arg contains "worker"
-    for arg in "$@"; do
-      case "$arg" in
-        *worker*) NEEDS_XVFB=true; break ;;
-      esac
-    done
-    ;;
-esac
-
-if [ "$NEEDS_XVFB" = "true" ]; then
   # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display
   export DISPLAY=:99
   export IAAI_HEADLESS=false
+
+  if [ -S /tmp/.X11-unix/X99 ] || [ -f /tmp/.X99-lock ]; then
+    echo "[entrypoint] Reusing existing Xvfb on DISPLAY=${DISPLAY}"
+    return 0
+  fi
+
   Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
   XVFB_PID=$!
   sleep 0.5
-  echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)"
+  echo "[entrypoint] Xvfb started (PID ${XVFB_PID}, DISPLAY=${DISPLAY})"
-fi
+}
 
-# Run Alembic migrations (only for API service, skip for worker/beat)
+start_proxy_bridge_if_needed "$@"
-case "$1" in
+start_xvfb_if_needed "$@"
-  uvicorn*)
-    echo "[entrypoint] Running Alembic migrations..."
-    alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..."
-    ;;
-esac
 
 exec "$@"