99 lines
3.2 KiB
Bash
99 lines
3.2 KiB
Bash
# Docker environment for mobile.de
|
|
|
|
# Database
|
|
POSTGRES_USER=mobilede
|
|
POSTGRES_PASSWORD=mobilede
|
|
POSTGRES_DB=mobilede_scraper
|
|
MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
|
|
MOBILEDE_DATABASE_ECHO=false
|
|
MOBILEDE_DATABASE_POOL_SIZE=20
|
|
MOBILEDE_DATABASE_MAX_OVERFLOW=40
|
|
MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
|
|
|
|
# Ports
|
|
MOBILEDE_HOST_API_PORT=8000
|
|
MOBILEDE_HOST_POSTGRES_PORT=5432
|
|
MOBILEDE_HOST_REDIS_PORT=6379
|
|
|
|
# Redis and Celery
|
|
MOBILEDE_REDIS_URL=redis://redis:6379/0
|
|
CELERY_BROKER_URL=redis://redis:6379/0
|
|
CELERY_RESULT_BACKEND=redis://redis:6379/0
|
|
CELERY_TASK_SOFT_TIME_LIMIT=86400
|
|
CELERY_TASK_TIME_LIMIT=86520
|
|
CELERY_BROKER_VISIBILITY_TIMEOUT=90000
|
|
CELERY_WORKER_CONCURRENCY=1
|
|
CELERY_WORKER_POOL=prefork
|
|
CELERY_WORKER_MAX_TASKS_PER_CHILD=5
|
|
|
|
# Runtime mode
|
|
MOBILEDE_STARTUP_SYNC_ENABLED=false
|
|
MOBILEDE_BEAT_SYNC_ENABLED=false
|
|
MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
|
|
MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
|
|
MOBILEDE_CONTINUOUS_SYNC_ENABLED=false
|
|
MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
|
|
MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
|
|
|
|
# Search settings
|
|
MOBILEDE_FILTERED_SEARCH_URL=https://www.mobile.de/ru/%D1%82%D1%80%D0%B0%D0%BD%D1%81%D0%BF%D0%BE%D1%80%D1%82%D0%BD%D1%8B%D0%B5-%D1%81%D1%80%D0%B5%D0%B4%D1%81%D1%82%D0%B2%D0%B0/%D0%BF%D0%BE%D0%B8%D1%81%D0%BA.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=20100&ms=22900&ms=14800&od=up&sb=rel&ref=dsp
|
|
MOBILEDE_FILTERED_SEARCH_URLS=
|
|
MOBILEDE_COMPLETE_SCOPE_BRANDS=BMW,Porsche,Skoda,Land Rover
|
|
MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED=false
|
|
MOBILEDE_REQUEST_DELAY_SECONDS=2.0
|
|
MOBILEDE_CONCURRENT_PAGES=1
|
|
MOBILEDE_HTTP_MAX_RETRIES=2
|
|
MOBILEDE_SEARCH_NETWORK_RETRY_DELAY_SECONDS=30
|
|
MOBILEDE_SOLD_PROBE_ANTIBOT_BACKOFF_SECONDS=1800
|
|
MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT=3
|
|
MOBILEDE_CURSOR_ENABLED=true
|
|
MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
|
|
MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
|
|
MOBILEDE_RUNTIME_INITIAL_TASKS=3
|
|
MOBILEDE_RESULTS_PER_PAGE=20
|
|
MOBILEDE_MAX_PAGE_NUMBER=50
|
|
MOBILEDE_SEGMENT_TARGET_RESULTS=1000
|
|
MOBILEDE_SKIP_EMPTY_WINDOW=true
|
|
MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS=true
|
|
MOBILEDE_COMPACT_SEGMENTS=true
|
|
MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=false
|
|
|
|
# Planner and overflow
|
|
MOBILEDE_PREPLAN_SEGMENT_PROBES=true
|
|
MOBILEDE_PREPLAN_MAX_SEGMENTS=700
|
|
MOBILEDE_PREPLAN_MAX_PROBES=80
|
|
MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
|
|
MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
|
|
MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
|
|
MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
|
|
MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP=true
|
|
MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
|
|
MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
|
|
|
|
# Existing car refresh
|
|
MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
|
|
MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
|
|
MOBILEDE_LIGHT_REFRESH_EXISTING=true
|
|
MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
|
|
MOBILEDE_DETAIL_QUEUE_ENABLED=false
|
|
MOBILEDE_DETAIL_QUEUE_MAX_PENDING=10000
|
|
MOBILEDE_DETAIL_CLAIM_TTL_SECONDS=300
|
|
MOBILEDE_DETAIL_MAX_RETRIES=8
|
|
MOBILEDE_DETAIL_MAX_PARSE_RETRIES=3
|
|
MOBILEDE_DETAIL_EXHAUSTED_TTL_SECONDS=604800
|
|
MOBILEDE_DETAIL_WORKER_CONCURRENCY=10
|
|
MOBILEDE_DETAIL_REQUEST_DELAY_SECONDS=0.08
|
|
MOBILEDE_DETAIL_RATE_LIMIT_SLOTS=1
|
|
MOBILEDE_HTTP_POOL_SIZE=32
|
|
|
|
# Proxy
|
|
MOBILEDE_PROXY_SERVER=
|
|
MOBILEDE_PROXY_USERNAME=
|
|
MOBILEDE_PROXY_PASSWORD=
|
|
SOCKS5_PROXY_HOST=
|
|
SOCKS5_PROXY_PORT=1002
|
|
SOCKS5_PROXY_USER=
|
|
SOCKS5_PROXY_PASS=
|
|
|
|
TZ=UTC
|