diff --git a/.dockerignore b/.dockerignore index 8a2cab3..acf045b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,5 +13,4 @@ coverage.xml .vscode/ *.egg-info/ dist/ -build/ -artifacts/ +build/ \ No newline at end of file diff --git a/.env.example b/.env.example index 8e87c4a..a8563ea 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,11 @@ IAAI_HEADLESS=true -IAAI_RAW_OUTPUT_JSON=iaai_raw_network.json IAAI_LOG_LEVEL=INFO # IAAI_LOG_FILE=iaai_scraper.log -# Conservative first-run mode. -IAAI_GENTLE_MODE=true +# Network capture settings. IAAI_CAPTURE_SAME_ORIGIN_ONLY=true IAAI_MAX_CAPTURED_REQUESTS=40 IAAI_MAX_CAPTURED_JSON_RESPONSES=30 -IAAI_WARM_SCROLL_ROUNDS=1 -IAAI_POST_OPEN_IDLE_MS=3500 # Sequential listing-first mode. IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars @@ -34,8 +30,7 @@ IAAI_BETWEEN_VEHICLES_MAX_S=6.0 IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0 IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0 -# Scheduler (default once per hour) -IAAI_SCHEDULER_INTERVAL_MINUTES=60 +# Sync settings IAAI_SYNC_ONLY_NEW=true # Retry / backoff @@ -50,6 +45,19 @@ IAAI_RETRY_JITTER_SECONDS=0.25 # IAAI_PROXY_USERNAME= # IAAI_PROXY_PASSWORD= -# Database. -IAAI_DATABASE_URL=sqlite:///iaai_scraper.db +# Database (PostgreSQL). +IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper IAAI_DATABASE_ECHO=false +IAAI_DATABASE_POOL_SIZE=5 +IAAI_DATABASE_MAX_OVERFLOW=10 + +# ─── Redis (Celery broker) ───────────────────────────────── +IAAI_REDIS_URL=redis://redis:6379/0 + +# ─── Celery ──────────────────────────────────────────────── +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 +CELERY_TASK_SOFT_TIME_LIMIT=600 +CELERY_TASK_TIME_LIMIT=900 +CELERY_WORKER_CONCURRENCY=1 +CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 diff --git a/.gitignore b/.gitignore index 62a835d..7755897 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,10 @@ coverage.xml *.pyd *.log *.db -*.json .env .vscode/ *.egg-info/ dist/ build/ +artifacts/ +celerybeat-schedule* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 822174a..f3df83d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,5 +17,6 @@ RUN chmod +x entrypoint.sh STOPSIGNAL SIGINT +# По умолчанию — API, но worker/beat переопределяют CMD в docker-compose ENTRYPOINT ["./entrypoint.sh"] -CMD ["python", "main.py", "run-daemon"] +CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker-compose.yml b/docker-compose.yml index 62b7299..750c37f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,105 @@ services: - iaai-scraper: + # ─── PostgreSQL ─────────────────────────────────────────── + postgres: + image: postgres:16-alpine + container_name: iaai-postgres + restart: unless-stopped + environment: + POSTGRES_USER: iaai + POSTGRES_PASSWORD: iaai + POSTGRES_DB: iaai_scraper + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"] + interval: 5s + timeout: 3s + retries: 5 + + # ─── Redis (Celery broker) ──────────────────────────────── + redis: + image: redis:7-alpine + container_name: iaai-redis + restart: unless-stopped + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + # ─── FastAPI ────────────────────────────────────────────── + api: build: . - container_name: iaai-scraper + container_name: iaai-api + restart: unless-stopped env_file: - path: .env required: false - volumes: - - ./:/app - - /app/.venv - - /app/__pycache__ - working_dir: /app + environment: + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper + IAAI_REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + ports: + - "8000:8000" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + command: > + uvicorn iaai_scraper.api.app:app + --host 0.0.0.0 --port 8000 --workers 2 + + # ─── Celery Worker ──────────────────────────────────────── + worker: + build: . + container_name: iaai-worker restart: unless-stopped - stop_grace_period: 30s - command: python main.py run-daemon + env_file: + - path: .env + required: false + environment: + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper + IAAI_REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + stop_grace_period: 60s + command: > + celery -A iaai_scraper.worker.celery_app worker + --loglevel=info --concurrency=1 --pool=prefork + -Q scraping --without-heartbeat + + # ─── Celery Beat (периодический планировщик) ────────────── + beat: + build: . + container_name: iaai-beat + restart: unless-stopped + env_file: + - path: .env + required: false + environment: + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper + IAAI_REDIS_URL: redis://redis:6379/0 + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + command: > + celery -A iaai_scraper.worker.celery_app beat + --loglevel=info + +volumes: + pgdata: diff --git a/entrypoint.sh b/entrypoint.sh index 9423a62..869b0cc 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -20,12 +20,38 @@ if [ -n "$SOCKS5_PROXY_HOST" ]; then echo "[entrypoint] Proxy bridge started (PID $BRIDGE_PID)" fi -# IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display -export DISPLAY=:99 -export IAAI_HEADLESS=false -Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp & -XVFB_PID=$! -sleep 0.5 -echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)" +# Xvfb needed only for worker (Playwright) — not for API or beat +NEEDS_XVFB=false +case "$1" in + celery*|python*main*) + NEEDS_XVFB=true + ;; + *) + # Check if any arg contains "worker" + for arg in "$@"; do + case "$arg" in + *worker*) NEEDS_XVFB=true; break ;; + esac + done + ;; +esac + +if [ "$NEEDS_XVFB" = "true" ]; then + # IAAI blocks headless Chromium on Linux; run headed via Xvfb virtual display + export DISPLAY=:99 + export IAAI_HEADLESS=false + Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp & + XVFB_PID=$! + sleep 0.5 + echo "[entrypoint] Xvfb started (PID $XVFB_PID, DISPLAY=$DISPLAY)" +fi + +# Run Alembic migrations (only for API service, skip for worker/beat) +case "$1" in + uvicorn*) + echo "[entrypoint] Running Alembic migrations..." + alembic upgrade head || echo "[entrypoint] WARNING: Alembic migration failed, continuing..." + ;; +esac exec "$@" diff --git a/requirements.txt b/requirements.txt index 847a1b3..f91a842 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,11 @@ playwright>=1.53.0 python-dotenv>=1.0.1 pydantic>=2.8.2 SQLAlchemy>=2.0.32 -PySocks>=1.7.1 +fastapi>=0.115.0 +uvicorn>=0.34.0 +celery>=5.4.0 +redis>=5.2.0 +psycopg2-binary>=2.9.9 +alembic>=1.14.0 pytest>=8.3.0 -pytest-cov>=5.0.0 +pytest-cov>=5.0.0 \ No newline at end of file