diff --git a/.env.example b/.env.example index 77802bf..9fa55cf 100644 --- a/.env.example +++ b/.env.example @@ -22,7 +22,7 @@ CELERY_RESULT_BACKEND=redis://redis:6379/0 CELERY_TASK_SOFT_TIME_LIMIT=86400 CELERY_TASK_TIME_LIMIT=86520 CELERY_BROKER_VISIBILITY_TIMEOUT=90000 -CELERY_WORKER_CONCURRENCY=3 +CELERY_WORKER_CONCURRENCY=1 CELERY_WORKER_POOL=prefork CELERY_WORKER_MAX_TASKS_PER_CHILD=5 @@ -36,15 +36,20 @@ MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600 MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1 # Search settings -MOBILEDE_FILTERED_SEARCH_URL= +MOBILEDE_FILTERED_SEARCH_URL=https://www.mobile.de/ru/%D1%82%D1%80%D0%B0%D0%BD%D1%81%D0%BF%D0%BE%D1%80%D1%82%D0%BD%D1%8B%D0%B5-%D1%81%D1%80%D0%B5%D0%B4%D1%81%D1%82%D0%B2%D0%B0/%D0%BF%D0%BE%D0%B8%D1%81%D0%BA.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=20100&ms=22900&ms=14800&od=up&sb=rel&ref=dsp MOBILEDE_FILTERED_SEARCH_URLS= -MOBILEDE_REQUEST_DELAY_SECONDS=0.30 +MOBILEDE_COMPLETE_SCOPE_BRANDS=BMW,Porsche,Skoda,Land Rover +MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED=false +MOBILEDE_REQUEST_DELAY_SECONDS=2.0 MOBILEDE_CONCURRENT_PAGES=1 +MOBILEDE_HTTP_MAX_RETRIES=2 +MOBILEDE_SEARCH_NETWORK_RETRY_DELAY_SECONDS=30 +MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS=1800 +MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT=3 MOBILEDE_CURSOR_ENABLED=true MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10 MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true MOBILEDE_RUNTIME_INITIAL_TASKS=3 -MOBILEDE_SEGMENT_PAGE_WINDOW=10 MOBILEDE_RESULTS_PER_PAGE=20 MOBILEDE_MAX_PAGE_NUMBER=50 MOBILEDE_SEGMENT_TARGET_RESULTS=1000 @@ -60,11 +65,8 @@ MOBILEDE_PREPLAN_MAX_PROBES=80 MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2 MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5 MOBILEDE_DYNAMIC_SEGMENT_PROBES=false -MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true -MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true -MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120 MOBILEDE_OVERFLOW_SPLIT_ENABLED=true -MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4 +MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP=true MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3 MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000 @@ -73,13 +75,16 @@ MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false MOBILEDE_INCREMENTAL_PAGE_WINDOW=10 MOBILEDE_LIGHT_REFRESH_EXISTING=true MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1 -MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false -MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1 -MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800 -MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45 -MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2 -MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2 -MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true +MOBILEDE_DETAIL_QUEUE_ENABLED=false +MOBILEDE_DETAIL_QUEUE_MAX_PENDING=10000 +MOBILEDE_DETAIL_CLAIM_TTL_SECONDS=300 +MOBILEDE_DETAIL_MAX_RETRIES=8 +MOBILEDE_DETAIL_MAX_PARSE_RETRIES=3 +MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS=604800 +MOBILEDE_DETAIL_WORKER_CONCURRENCY=10 +MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS=0.08 +MOBILEDE_DETAIL_RATE_LIMIT_SLOTS=1 +MOBILEDE_HTTP_POOL_SIZE=32 # Proxy MOBILEDE_PROXY_SERVER= diff --git a/docker-compose.yml b/docker-compose.yml index 25f3dc1..7078dbf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,45 +9,43 @@ CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} - CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-6} + CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-2} CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork} CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} - MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.45} - MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-1} + MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.08} + MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2} MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800" + MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS: ${MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS:-1800} + MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT: ${MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT:-3} MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-} MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-} + MOBILEDE_COMPLETE_SCOPE_BRANDS: ${MOBILEDE_COMPLETE_SCOPE_BRANDS:-} MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true} MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true} MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-3600} MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10} MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true} MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true} - MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-5} + MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2} MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true} - MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10} MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20} MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50} - MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-700} + MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-950} + MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH: ${MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH:-8} MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true} # Первый проход должен идти по заранее нарезанным leaf-сегментам. # Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально. MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true} MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400} - MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-320} + MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40} MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0} - MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-3} - MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN: ${MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN:-true} - MOBILEDE_FILTERED_URL_FAST_START: ${MOBILEDE_FILTERED_URL_FAST_START:-false} - MOBILEDE_REFINE_ADAPTIVE_SEGMENTS: ${MOBILEDE_REFINE_ADAPTIVE_SEGMENTS:-true} - MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS: ${MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS:-240} - MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES: ${MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES:-3} + MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-2} MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8} MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false} MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true} - MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.70} - MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.90} - MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.35} + MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.80} + MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.98} + MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.45} MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true} MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true} MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false} @@ -56,8 +54,17 @@ MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false} MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10} MOBILEDE_BEAT_ENRICH_ENABLED: ${MOBILEDE_BEAT_ENRICH_ENABLED:-true} - MOBILEDE_ENRICH_INTERVAL_SECONDS: ${MOBILEDE_ENRICH_INTERVAL_SECONDS:-900} - MOBILEDE_ENRICH_LOCK_TTL_SECONDS: ${MOBILEDE_ENRICH_LOCK_TTL_SECONDS:-300} + MOBILEDE_ENRICH_INTERVAL_SECONDS: ${MOBILEDE_ENRICH_INTERVAL_SECONDS:-60} + MOBILEDE_DETAIL_QUEUE_ENABLED: ${MOBILEDE_DETAIL_QUEUE_ENABLED:-true} + MOBILEDE_DETAIL_REFRESH_STALE_ENABLED: ${MOBILEDE_DETAIL_REFRESH_STALE_ENABLED:-false} + MOBILEDE_DETAIL_QUEUE_MAX_PENDING: ${MOBILEDE_DETAIL_QUEUE_MAX_PENDING:-1500} + MOBILEDE_DETAIL_CLAIM_TTL_SECONDS: ${MOBILEDE_DETAIL_CLAIM_TTL_SECONDS:-300} + MOBILEDE_DETAIL_MAX_RETRIES: ${MOBILEDE_DETAIL_MAX_RETRIES:-8} + MOBILEDE_DETAIL_MAX_PARSE_RETRIES: ${MOBILEDE_DETAIL_MAX_PARSE_RETRIES:-3} + MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS: ${MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS:-604800} + MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS: ${MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS:-0.08} + MOBILEDE_DETAIL_RATE_LIMIT_SLOTS: ${MOBILEDE_DETAIL_RATE_LIMIT_SLOTS:-1} + MOBILEDE_HTTP_POOL_SIZE: ${MOBILEDE_HTTP_POOL_SIZE:-32} MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true} MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} # Автовосстановление worker. @@ -182,12 +189,11 @@ services: stop_grace_period: 60s command: > celery -A mobilede_scraper.worker.celery_app worker - --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-8} --pool=${CELERY_WORKER_POOL:-prefork} + --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=${CELERY_WORKER_POOL:-prefork} --pidfile=/tmp/celery-worker.pid - -Q mobilede_sync --max-tasks-per-child=5 + -Q mobilede_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} healthcheck: - # Проверка worker. - test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"] + test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"] interval: 60s timeout: 10s retries: 3 @@ -214,9 +220,9 @@ services: stop_grace_period: 60s command: > celery -A mobilede_scraper.worker.celery_app worker - --loglevel=info --concurrency=1 --pool=${CELERY_WORKER_POOL:-prefork} + --loglevel=info --concurrency=${MOBILEDE_DETAIL_WORKER_CONCURRENCY:-10} --pool=${CELERY_WORKER_POOL:-prefork} --pidfile=/tmp/celery-images-worker.pid - -Q mobilede_images --max-tasks-per-child=20 + -Q mobilede_images --max-tasks-per-child=${MOBILEDE_DETAIL_MAX_TASKS_PER_CHILD:-500} healthcheck: test: ["CMD-SHELL", "test -f /tmp/celery-images-worker.pid && kill -0 $(cat /tmp/celery-images-worker.pid)"] interval: 60s diff --git a/entrypoint.sh b/entrypoint.sh index 84a9058..77606d1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -14,7 +14,7 @@ is_worker_command() { is_scrape_cli_command() { local joined="$*" case "$joined" in - *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*) + *"mobilede search"*|*"mobilede mobilede-search"*|*"mobilede detail"*|*"mobilede mobilede-detail"*|*"mobilede sync-search"*|*"mobilede sync-detail"*) return 0 ;; esac