update docker config

2026-04-16 18:01:16 +03:00
parent 4096d29a07
commit 32824dca5f
3 changed files with 51 additions and 125 deletions
--- a/12
+++ b/12
@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
+FROM python:3.12-slim

 ENV PYTHONDONTWRITEBYTECODE=1 \
    PYTHONUNBUFFERED=1
@@ -10,15 +10,11 @@ WORKDIR /app
 COPY pyproject.toml uv.lock ./
 RUN pip install --no-cache-dir uv \
    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
-    && pip install --no-cache-dir -r /tmp/requirements.txt \
-    && pip install --no-cache-dir playwright-stealth
-
-# Install Firefox browser (headless-friendly, bypasses Incapsula)
-RUN python -m playwright install firefox
+    && pip install --no-cache-dir -r /tmp/requirements.txt

 COPY . .
 RUN pip install --no-cache-dir -e .
-RUN python -m compileall -q iaai_scraper
+RUN python -m compileall -q encar_scraper
 RUN chmod +x entrypoint.sh
 RUN chown -R app:app /app

@@ -28,4 +24,4 @@ USER app

 # По умолчанию  API, но worker/beat переопределяют CMD в docker-compose
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "encar_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,26 +1,18 @@
-x-app-env: &app-env
-  # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
-  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
-  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
-  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
-  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
-  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300}
-  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600}
-  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
-  # Без лимита beat может забирать слишком большой объём за один запуск.
-  # Консервативный дефолт для anti-fraud: 300 авто за запуск.
-  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-300}
-  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
-  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
-  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
-  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
-  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
-  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
-  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
-  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  IAAI_BROWSER_ENGINE: ${IAAI_BROWSER_ENGINE:-auto}
+x-app-env: &app-env
+  # Database (отдельная БД для Encar, порт 5433)
+  ENCAR_DATABASE_URL: postgresql+psycopg2://encar:encar@postgres:5432/encar_db
+  ENCAR_DATABASE_POOL_RECYCLE_SECONDS: ${ENCAR_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  # Redis / Celery (порт 6380, db=1)
+  ENCAR_REDIS_URL: ${ENCAR_REDIS_URL:-redis://redis:6379/1}
+  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/1}
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/1}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-14400}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-14520}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-15000}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-50}
+  # Encar settings
+  ENCAR_BEAT_INTERVAL_MINUTES: ${ENCAR_BEAT_INTERVAL_MINUTES:-60}
+  ENCAR_BEAT_LIMIT: ${ENCAR_BEAT_LIMIT:-0}
  TZ: ${TZ:-UTC}

 x-env-file: &env-file
@@ -31,31 +23,23 @@ x-app-service: &app-service
  build: .
  env_file: *env-file
  environment: *app-env
-  volumes:
-    - ./runtime_config.json:/app/runtime_config.json:ro
-
-x-worker-service: &worker-service
-  <<: *app-service
-  volumes:
-    - ./runtime_config.json:/app/runtime_config.json:ro
-    - tokens_data:/data

 services:
-  # PostgreSQL
+  # PostgreSQL (порт 5434 наружу)
  postgres:
    image: postgres:16-alpine
-    container_name: iaai-postgres
+    container_name: encar-postgres
    restart: unless-stopped
    environment:
-      POSTGRES_USER: iaai
-      POSTGRES_PASSWORD: iaai
-      POSTGRES_DB: iaai_scraper
+      POSTGRES_USER: encar
+      POSTGRES_PASSWORD: encar
+      POSTGRES_DB: encar_db
    ports:
-      - "127.0.0.1:5432:5432"
+      - "127.0.0.1:5434:5432"
    volumes:
-      - pgdata:/var/lib/postgresql/data
+      - encar_pgdata:/var/lib/postgresql/data
    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
+      test: ["CMD-SHELL", "pg_isready -U encar -d encar_db"]
      interval: 5s
      timeout: 3s
      retries: 5
@@ -65,13 +49,13 @@ services:
        max-size: "10m"
        max-file: "5"

-  # Redis (Celery broker)
+  # Redis (порт 6380 наружу)
  redis:
    image: redis:7-alpine
-    container_name: iaai-redis
+    container_name: encar-redis
    restart: unless-stopped
    ports:
-      - "127.0.0.1:6379:6379"
+      - "127.0.0.1:6380:6379"
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
@@ -82,7 +66,7 @@ services:
      --appendonly yes
      --save 60 1000
    volumes:
-      - redisdata:/data
+      - encar_redisdata:/data
    logging:
      driver: json-file
      options:
@@ -92,7 +76,7 @@ services:
  # DB migrations
  migrate:
    <<: *app-service
-    container_name: iaai-migrate
+    container_name: encar-migrate
    restart: "no"
    depends_on:
      postgres:
@@ -102,24 +86,24 @@ services:
  # FastAPI
  api:
    <<: *app-service
-    container_name: iaai-api
+    container_name: encar-api
    restart: unless-stopped
    ports:
-      - "127.0.0.1:8000:8000"
+      - "127.0.0.1:8001:8000"
    depends_on:
      migrate:
        condition: service_completed_successfully
      redis:
        condition: service_healthy
    command: >
-      uvicorn iaai_scraper.api.app:app
+      uvicorn encar_scraper.api.app:app
      --host 0.0.0.0 --port 8000 --workers 1
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
      interval: 30s
      timeout: 10s
      retries: 3
-      start_period: 40s
+      start_period: 20s
    stop_grace_period: 30s
    logging:
      driver: json-file
@@ -127,10 +111,10 @@ services:
        max-size: "10m"
        max-file: "5"

-  # Celery Worker
+  # Celery Worker (Encar — только HTTP API, без браузера)
  worker:
-    <<: *worker-service
-    container_name: iaai-worker
+    <<: *app-service
+    container_name: encar-worker
    restart: unless-stopped
    init: true
    depends_on:
@@ -138,18 +122,18 @@ services:
        condition: service_completed_successfully
      redis:
        condition: service_healthy
-    stop_grace_period: 60s
+    stop_grace_period: 30s
    command: >
-      celery -A iaai_scraper.worker.celery_app worker
+      celery -A encar_scraper.worker.celery_app worker
      --loglevel=info --concurrency=4 --pool=prefork
      --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=5
+      -Q encar --max-tasks-per-child=50
    healthcheck:
      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
      interval: 60s
      timeout: 10s
      retries: 3
-      start_period: 90s
+      start_period: 30s
    logging:
      driver: json-file
      options:
@@ -158,8 +142,8 @@ services:

  # Celery Beat (периодический планировщик)
  beat:
-    <<: *worker-service
-    container_name: iaai-beat
+    <<: *app-service
+    container_name: encar-beat
    restart: unless-stopped
    init: true
    depends_on:
@@ -168,7 +152,7 @@ services:
      redis:
        condition: service_healthy
    command: >
-      celery -A iaai_scraper.worker.celery_app beat
+      celery -A encar_scraper.worker.celery_app beat
      --loglevel=info
      --pidfile=/tmp/celery-beat.pid
      --schedule=/tmp/celerybeat-schedule
@@ -177,7 +161,7 @@ services:
      interval: 60s
      timeout: 10s
      retries: 3
-      start_period: 90s
+      start_period: 30s
    logging:
      driver: json-file
      options:
@@ -185,6 +169,5 @@ services:
        max-file: "5"

 volumes:
-  pgdata:
-  redisdata:
-  tokens_data:
+  encar_pgdata:
+  encar_redisdata:
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -1,60 +1,7 @@
 #!/bin/bash
 set -euo pipefail

-is_worker_command() {
-  local joined="$*"
-  case "$joined" in
-    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
-      return 0
-      ;;
-  esac
-  return 1
-}
-
-is_scrape_cli_command() {
-  local joined="$*"
-  case "$joined" in
-    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
-      return 0
-      ;;
-  esac
-  return 1
-}
-
-needs_browser_runtime() {
-  is_worker_command "$@" || is_scrape_cli_command "$@"
-}
-
-start_proxy_bridge_if_needed() {
-  if ! needs_browser_runtime "$@"; then
-    return 0
-  fi
-
-  if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
-    return 0
-  fi
-
-  echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
-
-  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
-  fi
-
-  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
-  python -m iaai_scraper.proxy_bridge &
-  BRIDGE_PID=$!
-  sleep 1
-
-  if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
-    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
-    exit 1
-  fi
-
-  echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
-}
-
-start_proxy_bridge_if_needed "$@"
+# Encar scraper — простой entrypoint без браузера.
+# Просто запускает переданную команду.

 exec "$@"