qananasikq/encar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update docker config

Browse Source
This commit is contained in:
qananasikq
2026-04-16 18:01:16 +03:00
parent 4096d29a07
commit 32824dca5f
3 changed files with 51 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Dockerfile
View File

@@ -1,4 +1,4 @@
FROM mcr.microsoft.com/playwright/python:v1.58.0-noble FROM python:3.12-slim
ENV PYTHONDONTWRITEBYTECODE=1 \ ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 PYTHONUNBUFFERED=1
@@ -10,15 +10,11 @@ WORKDIR /app
COPY pyproject.toml uv.lock ./ COPY pyproject.toml uv.lock ./
RUN pip install --no-cache-dir uv \ RUN pip install --no-cache-dir uv \
&& uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \ && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
&& pip install --no-cache-dir -r /tmp/requirements.txt \ && pip install --no-cache-dir -r /tmp/requirements.txt
&& pip install --no-cache-dir playwright-stealth
# Install Firefox browser (headless-friendly, bypasses Incapsula)
RUN python -m playwright install firefox
COPY . . COPY . .
RUN pip install --no-cache-dir -e . RUN pip install --no-cache-dir -e .
RUN python -m compileall -q iaai_scraper RUN python -m compileall -q encar_scraper
RUN chmod +x entrypoint.sh RUN chmod +x entrypoint.sh
RUN chown -R app:app /app RUN chown -R app:app /app
@@ -28,4 +24,4 @@ USER app
# По умолчанию API, но worker/beat переопределяют CMD в docker-compose # По умолчанию API, но worker/beat переопределяют CMD в docker-compose
ENTRYPOINT ["./entrypoint.sh"] ENTRYPOINT ["./entrypoint.sh"]
CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] CMD ["uvicorn", "encar_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

107
docker-compose.yml
View File

@@ -1,26 +1,18 @@
x-app-env: &app-env x-app-env: &app-env
# NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here # Database (отдельная БД для Encar, порт 5433)
IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper ENCAR_DATABASE_URL: postgresql+psycopg2://encar:encar@postgres:5432/encar_db
IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0} ENCAR_DATABASE_POOL_RECYCLE_SECONDS: ${ENCAR_DATABASE_POOL_RECYCLE_SECONDS:-1800}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} # Redis / Celery (порт 6380, db=1)
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} ENCAR_REDIS_URL: ${ENCAR_REDIS_URL:-redis://redis:6379/1}
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/1}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300} CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/1}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600} CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-14400}
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200} CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-14520}
# Без лимита beat может забирать слишком большой объём за один запуск. CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-15000}
# Консервативный дефолт для anti-fraud: 300 авто за запуск. CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-50}
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-300} # Encar settings
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} ENCAR_BEAT_INTERVAL_MINUTES: ${ENCAR_BEAT_INTERVAL_MINUTES:-60}
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} ENCAR_BEAT_LIMIT: ${ENCAR_BEAT_LIMIT:-0}
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
IAAI_BROWSER_ENGINE: ${IAAI_BROWSER_ENGINE:-auto}
TZ: ${TZ:-UTC} TZ: ${TZ:-UTC}
x-env-file: &env-file x-env-file: &env-file
@@ -31,31 +23,23 @@ x-app-service: &app-service
build: . build: .
env_file: *env-file env_file: *env-file
environment: *app-env environment: *app-env
volumes:
- ./runtime_config.json:/app/runtime_config.json:ro
x-worker-service: &worker-service
<<: *app-service
volumes:
- ./runtime_config.json:/app/runtime_config.json:ro
- tokens_data:/data
services: services:
# PostgreSQL # PostgreSQL (порт 5434 наружу)
postgres: postgres:
image: postgres:16-alpine image: postgres:16-alpine
container_name: iaai-postgres container_name: encar-postgres
restart: unless-stopped restart: unless-stopped
environment: environment:
POSTGRES_USER: iaai POSTGRES_USER: encar
POSTGRES_PASSWORD: iaai POSTGRES_PASSWORD: encar
POSTGRES_DB: iaai_scraper POSTGRES_DB: encar_db
ports: ports:
- "127.0.0.1:5432:5432" - "127.0.0.1:5434:5432"
volumes: volumes:
- pgdata:/var/lib/postgresql/data - encar_pgdata:/var/lib/postgresql/data
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"] test: ["CMD-SHELL", "pg_isready -U encar -d encar_db"]
interval: 5s interval: 5s
timeout: 3s timeout: 3s
retries: 5 retries: 5
@@ -65,13 +49,13 @@ services:
max-size: "10m" max-size: "10m"
max-file: "5" max-file: "5"
# Redis (Celery broker) # Redis (порт 6380 наружу)
redis: redis:
image: redis:7-alpine image: redis:7-alpine
container_name: iaai-redis container_name: encar-redis
restart: unless-stopped restart: unless-stopped
ports: ports:
- "127.0.0.1:6379:6379" - "127.0.0.1:6380:6379"
healthcheck: healthcheck:
test: ["CMD", "redis-cli", "ping"] test: ["CMD", "redis-cli", "ping"]
interval: 5s interval: 5s
@@ -82,7 +66,7 @@ services:
--appendonly yes --appendonly yes
--save 60 1000 --save 60 1000
volumes: volumes:
- redisdata:/data - encar_redisdata:/data
logging: logging:
driver: json-file driver: json-file
options: options:
@@ -92,7 +76,7 @@ services:
# DB migrations # DB migrations
migrate: migrate:
<<: *app-service <<: *app-service
container_name: iaai-migrate container_name: encar-migrate
restart: "no" restart: "no"
depends_on: depends_on:
postgres: postgres:
@@ -102,24 +86,24 @@ services:
# FastAPI # FastAPI
api: api:
<<: *app-service <<: *app-service
container_name: iaai-api container_name: encar-api
restart: unless-stopped restart: unless-stopped
ports: ports:
- "127.0.0.1:8000:8000" - "127.0.0.1:8001:8000"
depends_on: depends_on:
migrate: migrate:
condition: service_completed_successfully condition: service_completed_successfully
redis: redis:
condition: service_healthy condition: service_healthy
command: > command: >
uvicorn iaai_scraper.api.app:app uvicorn encar_scraper.api.app:app
--host 0.0.0.0 --port 8000 --workers 1 --host 0.0.0.0 --port 8000 --workers 1
healthcheck: healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 3 retries: 3
start_period: 40s start_period: 20s
stop_grace_period: 30s stop_grace_period: 30s
logging: logging:
driver: json-file driver: json-file
@@ -127,10 +111,10 @@ services:
max-size: "10m" max-size: "10m"
max-file: "5" max-file: "5"
# Celery Worker # Celery Worker (Encar — только HTTP API, без браузера)
worker: worker:
<<: *worker-service <<: *app-service
container_name: iaai-worker container_name: encar-worker
restart: unless-stopped restart: unless-stopped
init: true init: true
depends_on: depends_on:
@@ -138,18 +122,18 @@ services:
condition: service_completed_successfully condition: service_completed_successfully
redis: redis:
condition: service_healthy condition: service_healthy
stop_grace_period: 60s stop_grace_period: 30s
command: > command: >
celery -A iaai_scraper.worker.celery_app worker celery -A encar_scraper.worker.celery_app worker
--loglevel=info --concurrency=4 --pool=prefork --loglevel=info --concurrency=4 --pool=prefork
--pidfile=/tmp/celery-worker.pid --pidfile=/tmp/celery-worker.pid
-Q scraping --max-tasks-per-child=5 -Q encar --max-tasks-per-child=50
healthcheck: healthcheck:
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"] test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
interval: 60s interval: 60s
timeout: 10s timeout: 10s
retries: 3 retries: 3
start_period: 90s start_period: 30s
logging: logging:
driver: json-file driver: json-file
options: options:
@@ -158,8 +142,8 @@ services:
# Celery Beat (периодический планировщик) # Celery Beat (периодический планировщик)
beat: beat:
<<: *worker-service <<: *app-service
container_name: iaai-beat container_name: encar-beat
restart: unless-stopped restart: unless-stopped
init: true init: true
depends_on: depends_on:
@@ -168,7 +152,7 @@ services:
redis: redis:
condition: service_healthy condition: service_healthy
command: > command: >
celery -A iaai_scraper.worker.celery_app beat celery -A encar_scraper.worker.celery_app beat
--loglevel=info --loglevel=info
--pidfile=/tmp/celery-beat.pid --pidfile=/tmp/celery-beat.pid
--schedule=/tmp/celerybeat-schedule --schedule=/tmp/celerybeat-schedule
@@ -177,7 +161,7 @@ services:
interval: 60s interval: 60s
timeout: 10s timeout: 10s
retries: 3 retries: 3
start_period: 90s start_period: 30s
logging: logging:
driver: json-file driver: json-file
options: options:
@@ -185,6 +169,5 @@ services:
max-file: "5" max-file: "5"
volumes: volumes:
pgdata: encar_pgdata:
redisdata: encar_redisdata:
tokens_data:

57
entrypoint.sh
View File

@@ -1,60 +1,7 @@
#!/bin/bash #!/bin/bash
set -euo pipefail set -euo pipefail
is_worker_command() { # Encar scraper — простой entrypoint без браузера.
local joined="$*" # Просто запускает переданную команду.
case "$joined" in
*"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
return 0
;;
esac
return 1
}
is_scrape_cli_command() {
local joined="$*"
case "$joined" in
*"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
return 0
;;
esac
return 1
}
needs_browser_runtime() {
is_worker_command "$@" || is_scrape_cli_command "$@"
}
start_proxy_bridge_if_needed() {
if ! needs_browser_runtime "$@"; then
return 0
fi
if [ -z "${SOCKS5_PROXY_HOST:-}" ]; then
return 0
fi
echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
fi
echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
python -m iaai_scraper.proxy_bridge &
BRIDGE_PID=$!
sleep 1
if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
exit 1
fi
echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
}
start_proxy_bridge_if_needed "$@"
exec "$@" exec "$@"