Update worker startup settings
This commit is contained in:
+20
-15
@@ -22,7 +22,7 @@ CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
CELERY_TASK_SOFT_TIME_LIMIT=86400
|
||||
CELERY_TASK_TIME_LIMIT=86520
|
||||
CELERY_BROKER_VISIBILITY_TIMEOUT=90000
|
||||
CELERY_WORKER_CONCURRENCY=3
|
||||
CELERY_WORKER_CONCURRENCY=1
|
||||
CELERY_WORKER_POOL=prefork
|
||||
CELERY_WORKER_MAX_TASKS_PER_CHILD=5
|
||||
|
||||
@@ -36,15 +36,20 @@ MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
|
||||
MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
|
||||
|
||||
# Search settings
|
||||
MOBILEDE_FILTERED_SEARCH_URL=
|
||||
MOBILEDE_FILTERED_SEARCH_URL=https://www.mobile.de/ru/%D1%82%D1%80%D0%B0%D0%BD%D1%81%D0%BF%D0%BE%D1%80%D1%82%D0%BD%D1%8B%D0%B5-%D1%81%D1%80%D0%B5%D0%B4%D1%81%D1%82%D0%B2%D0%B0/%D0%BF%D0%BE%D0%B8%D1%81%D0%BA.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=20100&ms=22900&ms=14800&od=up&sb=rel&ref=dsp
|
||||
MOBILEDE_FILTERED_SEARCH_URLS=
|
||||
MOBILEDE_REQUEST_DELAY_SECONDS=0.30
|
||||
MOBILEDE_COMPLETE_SCOPE_BRANDS=BMW,Porsche,Skoda,Land Rover
|
||||
MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED=false
|
||||
MOBILEDE_REQUEST_DELAY_SECONDS=2.0
|
||||
MOBILEDE_CONCURRENT_PAGES=1
|
||||
MOBILEDE_HTTP_MAX_RETRIES=2
|
||||
MOBILEDE_SEARCH_NETWORK_RETRY_DELAY_SECONDS=30
|
||||
MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS=1800
|
||||
MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT=3
|
||||
MOBILEDE_CURSOR_ENABLED=true
|
||||
MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
|
||||
MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
|
||||
MOBILEDE_RUNTIME_INITIAL_TASKS=3
|
||||
MOBILEDE_SEGMENT_PAGE_WINDOW=10
|
||||
MOBILEDE_RESULTS_PER_PAGE=20
|
||||
MOBILEDE_MAX_PAGE_NUMBER=50
|
||||
MOBILEDE_SEGMENT_TARGET_RESULTS=1000
|
||||
@@ -60,11 +65,8 @@ MOBILEDE_PREPLAN_MAX_PROBES=80
|
||||
MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
|
||||
MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
|
||||
MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
|
||||
MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true
|
||||
MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true
|
||||
MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120
|
||||
MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
|
||||
MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4
|
||||
MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP=true
|
||||
MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
|
||||
MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
|
||||
|
||||
@@ -73,13 +75,16 @@ MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
|
||||
MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
|
||||
MOBILEDE_LIGHT_REFRESH_EXISTING=true
|
||||
MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
|
||||
MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false
|
||||
MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1
|
||||
MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800
|
||||
MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45
|
||||
MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2
|
||||
MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2
|
||||
MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true
|
||||
MOBILEDE_DETAIL_QUEUE_ENABLED=false
|
||||
MOBILEDE_DETAIL_QUEUE_MAX_PENDING=10000
|
||||
MOBILEDE_DETAIL_CLAIM_TTL_SECONDS=300
|
||||
MOBILEDE_DETAIL_MAX_RETRIES=8
|
||||
MOBILEDE_DETAIL_MAX_PARSE_RETRIES=3
|
||||
MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS=604800
|
||||
MOBILEDE_DETAIL_WORKER_CONCURRENCY=10
|
||||
MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS=0.08
|
||||
MOBILEDE_DETAIL_RATE_LIMIT_SLOTS=1
|
||||
MOBILEDE_HTTP_POOL_SIZE=32
|
||||
|
||||
# Proxy
|
||||
MOBILEDE_PROXY_SERVER=
|
||||
|
||||
Reference in New Issue
Block a user