qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update scraper package

Browse Source
This commit is contained in:
qananasikq
2026-04-24 20:46:58 +03:00
parent 6aba4f0dbd
commit d484088e18
59 changed files with 2548 additions and 1272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
docker-compose.yml
View File

@@ -1,35 +1,40 @@
x-app-env: &app-env
# Всегда используем Postgres.
IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
# 0 = без лимита.
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true}
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
IAAI_BROWSER_ENGINE: chromium
DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3}
DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9}
DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8}
DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true}
DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia}
DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto}
DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/}
DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999}
DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000}
DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true}
# Работаем через Algolia; fallback в sitemap по умолчанию отключён.
DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false}
DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true}
DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json}
DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
DUBIZZLE_BROWSER_ENGINE: chromium
# Self-heal для worker.
IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true}
DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900}
DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
TZ: ${TZ:-UTC}
x-env-file: &env-file
@@ -42,29 +47,32 @@ x-app-service: &app-service
environment: *app-env
volumes:
- ./runtime_config.json:/app/runtime_config.json:ro
- ./artifacts:/app/artifacts
x-worker-service: &worker-service
<<: *app-service
volumes:
- ./runtime_config.json:/app/runtime_config.json:ro
- ./artifacts:/app/artifacts
- tokens_data:/data
services:
# PostgreSQL.
postgres:
image: postgres:16-alpine
container_name: iaai-postgres
container_name: dubizzle-postgres
restart: unless-stopped
environment:
POSTGRES_USER: iaai
POSTGRES_PASSWORD: iaai
POSTGRES_DB: iaai_scraper
POSTGRES_USER: dubizzle
POSTGRES_PASSWORD: dubizzle
POSTGRES_DB: dubizzle_scraper
ports:
- "127.0.0.1:5432:5432"
# Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
- "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432"
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"]
interval: 5s
timeout: 3s
retries: 5
@@ -77,10 +85,11 @@ services:
# Redis.
redis:
image: redis:7-alpine
container_name: iaai-redis
container_name: dubizzle-redis
restart: unless-stopped
ports:
- "127.0.0.1:6379:6379"
# Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
- "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
@@ -101,7 +110,7 @@ services:
# Миграции.
migrate:
<<: *app-service
container_name: iaai-migrate
container_name: dubizzle-migrate
restart: "no"
depends_on:
postgres:
@@ -111,17 +120,18 @@ services:
# API.
api:
<<: *app-service
container_name: iaai-api
container_name: dubizzle-api
restart: unless-stopped
ports:
- "127.0.0.1:8000:8000"
# Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
- "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000"
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
command: >
uvicorn iaai_scraper.api.app:app
uvicorn dubizzle_scraper.api.app:app
--host 0.0.0.0 --port 8000 --workers 1
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -148,13 +158,13 @@ services:
condition: service_healthy
stop_grace_period: 60s
command: >
celery -A iaai_scraper.worker.celery_app worker
celery -A dubizzle_scraper.worker.celery_app worker
--loglevel=info --concurrency=1 --pool=solo
--pidfile=/tmp/celery-worker.pid
-Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
healthcheck:
# Проверка worker.
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"]
interval: 60s
timeout: 10s
retries: 3
@@ -168,7 +178,7 @@ services:
# Beat.
beat:
<<: *worker-service
container_name: iaai-beat
container_name: dubizzle-beat
restart: unless-stopped
init: true
depends_on:
@@ -177,7 +187,7 @@ services:
redis:
condition: service_healthy
command: >
celery -A iaai_scraper.worker.celery_app beat
celery -A dubizzle_scraper.worker.celery_app beat
--loglevel=info
--pidfile=/tmp/celery-beat.pid
--schedule=/tmp/celerybeat-schedule
@@ -193,6 +203,25 @@ services:
max-size: "10m"
max-file: "5"
# Разовый прогон sync-listing через Docker CLI.
sync:
<<: *worker-service
container_name: dubizzle-sync
restart: "no"
profiles: ["manual"]
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
command: >
bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json"
logging:
driver: json-file
options:
max-size: "10m"
max-file: "5"
volumes:
pgdata:
redisdata: