postgres redis docker compose setup

2026-04-08 22:54:51 +03:00
parent f4b9d20191
commit e5700f9623
7 changed files with 161 additions and 31 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -13,5 +13,4 @@ coverage.xml
 .vscode/
 *.egg-info/
 dist/
-build/
-artifacts/
+build/
--- a/.env.example
+++ b/.env.example
@@ -1,15 +1,11 @@
 IAAI_HEADLESS=true
-IAAI_RAW_OUTPUT_JSON=iaai_raw_network.json
 IAAI_LOG_LEVEL=INFO
 # IAAI_LOG_FILE=iaai_scraper.log

-# Conservative first-run mode.
-IAAI_GENTLE_MODE=true
+# Network capture settings.
 IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
 IAAI_MAX_CAPTURED_REQUESTS=40
 IAAI_MAX_CAPTURED_JSON_RESPONSES=30
-IAAI_WARM_SCROLL_ROUNDS=1
-IAAI_POST_OPEN_IDLE_MS=3500

 # Sequential listing-first mode.
 IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
@@ -34,8 +30,7 @@ IAAI_BETWEEN_VEHICLES_MAX_S=6.0
 IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
 IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0

-# Scheduler (default once per hour)
-IAAI_SCHEDULER_INTERVAL_MINUTES=60
+# Sync settings
 IAAI_SYNC_ONLY_NEW=true

 # Retry / backoff
@@ -50,6 +45,19 @@ IAAI_RETRY_JITTER_SECONDS=0.25
 # IAAI_PROXY_USERNAME=
 # IAAI_PROXY_PASSWORD=

-# Database.
-IAAI_DATABASE_URL=sqlite:///iaai_scraper.db
+# Database (PostgreSQL).
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
 IAAI_DATABASE_ECHO=false
+IAAI_DATABASE_POOL_SIZE=5
+IAAI_DATABASE_MAX_OVERFLOW=10
+
+# ─── Redis (Celery broker) ─────────────────────────────────
+IAAI_REDIS_URL=redis://redis:6379/0
+
+# ─── Celery ────────────────────────────────────────────────
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CELERY_TASK_SOFT_TIME_LIMIT=600
+CELERY_TASK_TIME_LIMIT=900
+CELERY_WORKER_CONCURRENCY=1
+CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
--- a/.gitignore
+++ b/.gitignore
@@ -8,9 +8,10 @@ coverage.xml
 *.pyd
 *.log
 *.db
-*.json
 .env
 .vscode/
 *.egg-info/
 dist/
 build/
+artifacts/
+celerybeat-schedule*
--- a/3
+++ b/3
@@ -17,5 +17,6 @@ RUN chmod +x entrypoint.sh

 STOPSIGNAL SIGINT

+# По умолчанию — API, но worker/beat переопределяют CMD в docker-compose
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["python", "main.py", "run-daemon"]
+CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,15 +1,105 @@
 services:
-  iaai-scraper:
+  # ─── PostgreSQL ───────────────────────────────────────────
+  postgres:
+    image: postgres:16-alpine
+    container_name: iaai-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: iaai
+      POSTGRES_PASSWORD: iaai
+      POSTGRES_DB: iaai_scraper
+    ports:
+      - "5432:5432"
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  # ─── Redis (Celery broker) ────────────────────────────────
+  redis:
+    image: redis:7-alpine
+    container_name: iaai-redis
+    restart: unless-stopped
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+
+  # ─── FastAPI ──────────────────────────────────────────────
+  api:
    build: .
-    container_name: iaai-scraper
+    container_name: iaai-api
+    restart: unless-stopped
    env_file:
      - path: .env
        required: false
-    volumes:
-      - ./:/app
-      - /app/.venv
-      - /app/__pycache__
-    working_dir: /app
+    environment:
+      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+      IAAI_REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+    ports:
+      - "8000:8000"
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    command: >
+      uvicorn iaai_scraper.api.app:app
+      --host 0.0.0.0 --port 8000 --workers 2
+
+  # ─── Celery Worker ────────────────────────────────────────
+  worker:
+    build: .
+    container_name: iaai-worker
    restart: unless-stopped
-    stop_grace_period: 30s
-    command: python main.py run-daemon
+    env_file:
+      - path: .env
+        required: false
+    environment:
+      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+      IAAI_REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    stop_grace_period: 60s
+    command: >
+      celery -A iaai_scraper.worker.celery_app worker
+      --loglevel=info --concurrency=1 --pool=prefork
+      -Q scraping --without-heartbeat
+
+  # ─── Celery Beat (периодический планировщик) ──────────────
+  beat:
+    build: .
+    container_name: iaai-beat
+    restart: unless-stopped
+    env_file:
+      - path: .env
+        required: false
+    environment:
+      IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+      IAAI_REDIS_URL: redis://redis:6379/0
+      CELERY_BROKER_URL: redis://redis:6379/0
+      CELERY_RESULT_BACKEND: redis://redis:6379/0
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    command: >
+      celery -A iaai_scraper.worker.celery_app beat
+      --loglevel=info
+
+volumes:
+  pgdata:
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -20,12 +20,38 @@ if [ -n "$SOCKS5_PROXY_HOST" ]; then
  echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)"
 fi

-# IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display
-export DISPLAY=:99
-export IAAI_HEADLESS=false
-Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
-XVFB_PID=$!
-sleep 0.5
-echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)"
+# Xvfb needed only for worker (Playwright) — not for API or beat
+NEEDS_XVFB=false
+case "$1" in
+  celery*|python*main*)
+    NEEDS_XVFB=true
+    ;;
+  *)
+    # Check if any arg contains "worker"
+    for arg in "$@"; do
+      case "$arg" in
+        *worker*) NEEDS_XVFB=true; break ;;
+      esac
+    done
+    ;;
+esac
+
+if [ "$NEEDS_XVFB" = "true" ]; then
+  # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display
+  export DISPLAY=:99
+  export IAAI_HEADLESS=false
+  Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp &
+  XVFB_PID=$!
+  sleep 0.5
+  echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)"
+fi
+
+# Run Alembic migrations (only for API service, skip for worker/beat)
+case "$1" in
+  uvicorn*)
+    echo "[entrypoint] Running Alembic migrations..."
+    alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..."
+    ;;
+esac

 exec "$@"
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,6 +2,11 @@ playwright>=1.53.0
 python-dotenv>=1.0.1
 pydantic>=2.8.2
 SQLAlchemy>=2.0.32
-PySocks>=1.7.1
+fastapi>=0.115.0
+uvicorn>=0.34.0
+celery>=5.4.0
+redis>=5.2.0
+psycopg2-binary>=2.9.9
+alembic>=1.14.0
 pytest>=8.3.0
-pytest-cov>=5.0.0
+pytest-cov>=5.0.0