qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fast parser

Browse Source
This commit is contained in:
qananasikq
2026-04-24 12:39:05 +03:00
parent 3a3222c7dc
commit fc1bea562a
19 changed files with 2482 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
docker-compose.yml
View File

@@ -5,23 +5,38 @@ x-app-env: &app-env
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
IAAI_DATABASE_POOL_SIZE: ${IAAI_DATABASE_POOL_SIZE:-20}
IAAI_DATABASE_MAX_OVERFLOW: ${IAAI_DATABASE_MAX_OVERFLOW:-40}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
IAAI_PROFILE: ${IAAI_PROFILE:-fast}
IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}}
IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true}
IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false}
IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false}
IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1}
IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-2}
IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0}
IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1}
IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
# 0 = без лимита.
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000}
IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-32}
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-runtime}
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json}
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
IAAI_BROWSER_ENGINE: chromium
# Self-heal для worker.
@@ -30,6 +45,8 @@ x-app-env: &app-env
IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
# Если в Postgres нет записей дольше 60 минут при активной работе — полный restart с segment 1.
IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
TZ: ${TZ:-UTC}
x-env-file: &env-file
@@ -149,9 +166,9 @@ services:
stop_grace_period: 60s
command: >
celery -A iaai_scraper.worker.celery_app worker
--loglevel=info --concurrency=1 --pool=solo
--loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork}
--pidfile=/tmp/celery-worker.pid
-Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-Q iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
healthcheck:
# Проверка worker.
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]