diff --git a/Dockerfile b/Dockerfile index 45b69b9..a298308 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/playwright/python:v1.58.0-noble +FROM python:3.12-slim ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 @@ -10,15 +10,11 @@ WORKDIR /app COPY pyproject.toml uv.lock ./ RUN pip install --no-cache-dir uv \ && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \ - && pip install --no-cache-dir -r /tmp/requirements.txt \ - && pip install --no-cache-dir playwright-stealth - -# Install Firefox browser (headless-friendly, bypasses Incapsula) -RUN python -m playwright install firefox + && pip install --no-cache-dir -r /tmp/requirements.txt COPY . . RUN pip install --no-cache-dir -e . -RUN python -m compileall -q iaai_scraper +RUN python -m compileall -q encar_scraper RUN chmod +x entrypoint.sh RUN chown -R app:app /app @@ -28,4 +24,4 @@ USER app # По умолчанию API, но worker/beat переопределяют CMD в docker-compose ENTRYPOINT ["./entrypoint.sh"] -CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "encar_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker-compose.yml b/docker-compose.yml index 93cc784..47bfaf5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,26 +1,18 @@ -x-app-env: &app-env - # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here - IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper - IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0} - CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} - CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} - IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800} - CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300} - CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600} - CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200} - # Без лимита beat может забирать слишком большой объём за один запуск. - # Консервативный дефолт для anti-fraud: 300 авто за запуск. - CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-300} - CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} - CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} - IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40} - IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true} - IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999} - IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000} - IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true} - IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json} - IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} - IAAI_BROWSER_ENGINE: ${IAAI_BROWSER_ENGINE:-auto} +x-app-env: &app-env + # Database (отдельная БД для Encar, порт 5433) + ENCAR_DATABASE_URL: postgresql+psycopg2://encar:encar@postgres:5432/encar_db + ENCAR_DATABASE_POOL_RECYCLE_SECONDS: ${ENCAR_DATABASE_POOL_RECYCLE_SECONDS:-1800} + # Redis / Celery (порт 6380, db=1) + ENCAR_REDIS_URL: ${ENCAR_REDIS_URL:-redis://redis:6379/1} + CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/1} + CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/1} + CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-14400} + CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-14520} + CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-15000} + CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-50} + # Encar settings + ENCAR_BEAT_INTERVAL_MINUTES: ${ENCAR_BEAT_INTERVAL_MINUTES:-60} + ENCAR_BEAT_LIMIT: ${ENCAR_BEAT_LIMIT:-0} TZ: ${TZ:-UTC} x-env-file: &env-file @@ -31,31 +23,23 @@ x-app-service: &app-service build: . env_file: *env-file environment: *app-env - volumes: - - ./runtime_config.json:/app/runtime_config.json:ro - -x-worker-service: &worker-service - <<: *app-service - volumes: - - ./runtime_config.json:/app/runtime_config.json:ro - - tokens_data:/data services: - # PostgreSQL + # PostgreSQL (порт 5434 наружу) postgres: image: postgres:16-alpine - container_name: iaai-postgres + container_name: encar-postgres restart: unless-stopped environment: - POSTGRES_USER: iaai - POSTGRES_PASSWORD: iaai - POSTGRES_DB: iaai_scraper + POSTGRES_USER: encar + POSTGRES_PASSWORD: encar + POSTGRES_DB: encar_db ports: - - "127.0.0.1:5432:5432" + - "127.0.0.1:5434:5432" volumes: - - pgdata:/var/lib/postgresql/data + - encar_pgdata:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"] + test: ["CMD-SHELL", "pg_isready -U encar -d encar_db"] interval: 5s timeout: 3s retries: 5 @@ -65,13 +49,13 @@ services: max-size: "10m" max-file: "5" - # Redis (Celery broker) + # Redis (порт 6380 наружу) redis: image: redis:7-alpine - container_name: iaai-redis + container_name: encar-redis restart: unless-stopped ports: - - "127.0.0.1:6379:6379" + - "127.0.0.1:6380:6379" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s @@ -82,7 +66,7 @@ services: --appendonly yes --save 60 1000 volumes: - - redisdata:/data + - encar_redisdata:/data logging: driver: json-file options: @@ -92,7 +76,7 @@ services: # DB migrations migrate: <<: *app-service - container_name: iaai-migrate + container_name: encar-migrate restart: "no" depends_on: postgres: @@ -102,24 +86,24 @@ services: # FastAPI api: <<: *app-service - container_name: iaai-api + container_name: encar-api restart: unless-stopped ports: - - "127.0.0.1:8000:8000" + - "127.0.0.1:8001:8000" depends_on: migrate: condition: service_completed_successfully redis: condition: service_healthy command: > - uvicorn iaai_scraper.api.app:app + uvicorn encar_scraper.api.app:app --host 0.0.0.0 --port 8000 --workers 1 healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] interval: 30s timeout: 10s retries: 3 - start_period: 40s + start_period: 20s stop_grace_period: 30s logging: driver: json-file @@ -127,10 +111,10 @@ services: max-size: "10m" max-file: "5" - # Celery Worker + # Celery Worker (Encar — только HTTP API, без браузера) worker: - <<: *worker-service - container_name: iaai-worker + <<: *app-service + container_name: encar-worker restart: unless-stopped init: true depends_on: @@ -138,18 +122,18 @@ services: condition: service_completed_successfully redis: condition: service_healthy - stop_grace_period: 60s + stop_grace_period: 30s command: > - celery -A iaai_scraper.worker.celery_app worker + celery -A encar_scraper.worker.celery_app worker --loglevel=info --concurrency=4 --pool=prefork --pidfile=/tmp/celery-worker.pid - -Q scraping --max-tasks-per-child=5 + -Q encar --max-tasks-per-child=50 healthcheck: test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"] interval: 60s timeout: 10s retries: 3 - start_period: 90s + start_period: 30s logging: driver: json-file options: @@ -158,8 +142,8 @@ services: # Celery Beat (периодический планировщик) beat: - <<: *worker-service - container_name: iaai-beat + <<: *app-service + container_name: encar-beat restart: unless-stopped init: true depends_on: @@ -168,7 +152,7 @@ services: redis: condition: service_healthy command: > - celery -A iaai_scraper.worker.celery_app beat + celery -A encar_scraper.worker.celery_app beat --loglevel=info --pidfile=/tmp/celery-beat.pid --schedule=/tmp/celerybeat-schedule @@ -177,7 +161,7 @@ services: interval: 60s timeout: 10s retries: 3 - start_period: 90s + start_period: 30s logging: driver: json-file options: @@ -185,6 +169,5 @@ services: max-file: "5" volumes: - pgdata: - redisdata: - tokens_data: + encar_pgdata: + encar_redisdata: diff --git a/entrypoint.sh b/entrypoint.sh index 078cf3f..e5dcf8b 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,60 +1,7 @@ #!/bin/bash set -euo pipefail -is_worker_command() { - local joined="$*" - case "$joined" in - *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*) - return 0 - ;; - esac - return 1 -} - -is_scrape_cli_command() { - local joined="$*" - case "$joined" in - *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*) - return 0 - ;; - esac - return 1 -} - -needs_browser_runtime() { - is_worker_command "$@" || is_scrape_cli_command "$@" -} - -start_proxy_bridge_if_needed() { - if ! needs_browser_runtime "$@"; then - return 0 - fi - - if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then - return 0 - fi - - echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..." - - if [ -z "${IAAI_PROXY_SERVER:-}" ]; then - export IAAI_PROXY_SERVER="http://127.0.0.1:8899" - elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then - export IAAI_PROXY_SERVER="http://127.0.0.1:8899" - fi - - echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}" - python -m iaai_scraper.proxy_bridge & - BRIDGE_PID=$! - sleep 1 - - if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then - echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start" - exit 1 - fi - - echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})" -} - -start_proxy_bridge_if_needed "$@" +# Encar scraper — простой entrypoint без браузера. +# Просто запускает переданную команду. exec "$@"