Update worker startup settings

.env.example

@@ -22,7 +22,7 @@ CELERY_RESULT_BACKEND=redis://redis:6379/0
 CELERY_TASK_SOFT_TIME_LIMIT=86400
 CELERY_TASK_TIME_LIMIT=86520
 CELERY_BROKER_VISIBILITY_TIMEOUT=90000
-CELERY_WORKER_CONCURRENCY=3
+CELERY_WORKER_CONCURRENCY=1
 CELERY_WORKER_POOL=prefork
 CELERY_WORKER_MAX_TASKS_PER_CHILD=5
 
@@ -36,15 +36,20 @@ MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
 MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
 
 # Search settings
-MOBILEDE_FILTERED_SEARCH_URL=
+MOBILEDE_FILTERED_SEARCH_URL=https://www.mobile.de/ru/%D1%82%D1%80%D0%B0%D0%BD%D1%81%D0%BF%D0%BE%D1%80%D1%82%D0%BD%D1%8B%D0%B5-%D1%81%D1%80%D0%B5%D0%B4%D1%81%D1%82%D0%B2%D0%B0/%D0%BF%D0%BE%D0%B8%D1%81%D0%BA.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=20100&ms=22900&ms=14800&od=up&sb=rel&ref=dsp
 MOBILEDE_FILTERED_SEARCH_URLS=
-MOBILEDE_REQUEST_DELAY_SECONDS=0.30
+MOBILEDE_COMPLETE_SCOPE_BRANDS=BMW,Porsche,Skoda,Land Rover
+MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED=false
+MOBILEDE_REQUEST_DELAY_SECONDS=2.0
 MOBILEDE_CONCURRENT_PAGES=1
+MOBILEDE_HTTP_MAX_RETRIES=2
+MOBILEDE_SEARCH_NETWORK_RETRY_DELAY_SECONDS=30
+MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS=1800
+MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT=3
 MOBILEDE_CURSOR_ENABLED=true
 MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
 MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
 MOBILEDE_RUNTIME_INITIAL_TASKS=3
-MOBILEDE_SEGMENT_PAGE_WINDOW=10
 MOBILEDE_RESULTS_PER_PAGE=20
 MOBILEDE_MAX_PAGE_NUMBER=50
 MOBILEDE_SEGMENT_TARGET_RESULTS=1000
@@ -60,11 +65,8 @@ MOBILEDE_PREPLAN_MAX_PROBES=80
 MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
 MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
 MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
-MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true
-MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true
-MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120
 MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
-MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4
+MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP=true
 MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
 MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
 
@@ -73,13 +75,16 @@ MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
 MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
 MOBILEDE_LIGHT_REFRESH_EXISTING=true
 MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
-MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false
+MOBILEDE_DETAIL_QUEUE_ENABLED=false
-MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1
+MOBILEDE_DETAIL_QUEUE_MAX_PENDING=10000
-MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800
+MOBILEDE_DETAIL_CLAIM_TTL_SECONDS=300
-MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45
+MOBILEDE_DETAIL_MAX_RETRIES=8
-MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2
+MOBILEDE_DETAIL_MAX_PARSE_RETRIES=3
-MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2
+MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS=604800
-MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true
+MOBILEDE_DETAIL_WORKER_CONCURRENCY=10
+MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS=0.08
+MOBILEDE_DETAIL_RATE_LIMIT_SLOTS=1
+MOBILEDE_HTTP_POOL_SIZE=32
 
 # Proxy
 MOBILEDE_PROXY_SERVER=

docker-compose.yml

@@ -9,45 +9,43 @@
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-6}
+  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-2}
   CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
   CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.45}
+  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.08}
-  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-1}
+  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
   MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800"
+  MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS: ${MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS:-1800}
+  MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT: ${MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT:-3}
   MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-}
   MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-}
+  MOBILEDE_COMPLETE_SCOPE_BRANDS: ${MOBILEDE_COMPLETE_SCOPE_BRANDS:-}
   MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
   MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true}
   MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-3600}
   MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10}
   MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
   MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
-  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-5}
+  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2}
   MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true}
-  MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10}
   MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20}
   MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50}
-  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-700}
+  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-950}
+  MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH: ${MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH:-8}
   MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
   # Первый проход должен идти по заранее нарезанным leaf-сегментам.
   # Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально.
   MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true}
   MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400}
-  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-320}
+  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40}
   MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0}
-  MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-3}
+  MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-2}
-  MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN: ${MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN:-true}
-  MOBILEDE_FILTERED_URL_FAST_START: ${MOBILEDE_FILTERED_URL_FAST_START:-false}
-  MOBILEDE_REFINE_ADAPTIVE_SEGMENTS: ${MOBILEDE_REFINE_ADAPTIVE_SEGMENTS:-true}
-  MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS: ${MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS:-240}
-  MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES: ${MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES:-3}
   MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8}
   MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
   MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true}
-  MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.70}
+  MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.80}
-  MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.90}
+  MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.98}
-  MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.35}
+  MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.45}
   MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
   MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
   MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false}
@@ -56,8 +54,17 @@
   MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false}
   MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10}
   MOBILEDE_BEAT_ENRICH_ENABLED: ${MOBILEDE_BEAT_ENRICH_ENABLED:-true}
-  MOBILEDE_ENRICH_INTERVAL_SECONDS: ${MOBILEDE_ENRICH_INTERVAL_SECONDS:-900}
+  MOBILEDE_ENRICH_INTERVAL_SECONDS: ${MOBILEDE_ENRICH_INTERVAL_SECONDS:-60}
-  MOBILEDE_ENRICH_LOCK_TTL_SECONDS: ${MOBILEDE_ENRICH_LOCK_TTL_SECONDS:-300}
+  MOBILEDE_DETAIL_QUEUE_ENABLED: ${MOBILEDE_DETAIL_QUEUE_ENABLED:-true}
+  MOBILEDE_DETAIL_REFRESH_STALE_ENABLED: ${MOBILEDE_DETAIL_REFRESH_STALE_ENABLED:-false}
+  MOBILEDE_DETAIL_QUEUE_MAX_PENDING: ${MOBILEDE_DETAIL_QUEUE_MAX_PENDING:-1500}
+  MOBILEDE_DETAIL_CLAIM_TTL_SECONDS: ${MOBILEDE_DETAIL_CLAIM_TTL_SECONDS:-300}
+  MOBILEDE_DETAIL_MAX_RETRIES: ${MOBILEDE_DETAIL_MAX_RETRIES:-8}
+  MOBILEDE_DETAIL_MAX_PARSE_RETRIES: ${MOBILEDE_DETAIL_MAX_PARSE_RETRIES:-3}
+  MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS: ${MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS:-604800}
+  MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS: ${MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS:-0.08}
+  MOBILEDE_DETAIL_RATE_LIMIT_SLOTS: ${MOBILEDE_DETAIL_RATE_LIMIT_SLOTS:-1}
+  MOBILEDE_HTTP_POOL_SIZE: ${MOBILEDE_HTTP_POOL_SIZE:-32}
   MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true}
   MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
   # Автовосстановление worker.
@@ -182,12 +189,11 @@ services:
     stop_grace_period: 60s
     command: >
       celery -A mobilede_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-8} --pool=${CELERY_WORKER_POOL:-prefork}
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=${CELERY_WORKER_POOL:-prefork}
       --pidfile=/tmp/celery-worker.pid
-      -Q mobilede_sync --max-tasks-per-child=5
+      -Q mobilede_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
     healthcheck:
-      # Проверка worker.
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"]
       interval: 60s
       timeout: 10s
       retries: 3
@@ -214,9 +220,9 @@ services:
     stop_grace_period: 60s
     command: >
       celery -A mobilede_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=1 --pool=${CELERY_WORKER_POOL:-prefork}
+      --loglevel=info --concurrency=${MOBILEDE_DETAIL_WORKER_CONCURRENCY:-10} --pool=${CELERY_WORKER_POOL:-prefork}
       --pidfile=/tmp/celery-images-worker.pid
-      -Q mobilede_images --max-tasks-per-child=20
+      -Q mobilede_images --max-tasks-per-child=${MOBILEDE_DETAIL_MAX_TASKS_PER_CHILD:-500}
     healthcheck:
       test: ["CMD-SHELL", "test -f /tmp/celery-images-worker.pid && kill -0 $(cat /tmp/celery-images-worker.pid)"]
       interval: 60s

entrypoint.sh

@@ -14,7 +14,7 @@ is_worker_command() {
 is_scrape_cli_command() {
   local joined="$*"
   case "$joined" in
-    *"collect-listing"*|*"scrape-vehicle"*|*"sync-vehicle"*|*"sync-listing"*)
+    *"mobilede search"*|*"mobilede mobilede-search"*|*"mobilede detail"*|*"mobilede mobilede-detail"*|*"mobilede sync-search"*|*"mobilede sync-detail"*)
       return 0
       ;;
   esac