94 lines
2.7 KiB
Plaintext
94 lines
2.7 KiB
Plaintext
# Docker environment for mobile.de
|
|
|
|
# Database
|
|
POSTGRES_USER=mobilede
|
|
POSTGRES_PASSWORD=mobilede
|
|
POSTGRES_DB=mobilede_scraper
|
|
MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
|
|
MOBILEDE_DATABASE_ECHO=false
|
|
MOBILEDE_DATABASE_POOL_SIZE=20
|
|
MOBILEDE_DATABASE_MAX_OVERFLOW=40
|
|
MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
|
|
|
|
# Ports
|
|
MOBILEDE_HOST_API_PORT=8000
|
|
MOBILEDE_HOST_POSTGRES_PORT=5432
|
|
MOBILEDE_HOST_REDIS_PORT=6379
|
|
|
|
# Redis and Celery
|
|
MOBILEDE_REDIS_URL=redis://redis:6379/0
|
|
CELERY_BROKER_URL=redis://redis:6379/0
|
|
CELERY_RESULT_BACKEND=redis://redis:6379/0
|
|
CELERY_TASK_SOFT_TIME_LIMIT=86400
|
|
CELERY_TASK_TIME_LIMIT=86520
|
|
CELERY_BROKER_VISIBILITY_TIMEOUT=90000
|
|
CELERY_WORKER_CONCURRENCY=3
|
|
CELERY_WORKER_POOL=prefork
|
|
CELERY_WORKER_MAX_TASKS_PER_CHILD=5
|
|
|
|
# Runtime mode
|
|
MOBILEDE_STARTUP_SYNC_ENABLED=false
|
|
MOBILEDE_BEAT_SYNC_ENABLED=false
|
|
MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
|
|
MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
|
|
MOBILEDE_CONTINUOUS_SYNC_ENABLED=false
|
|
MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
|
|
MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
|
|
|
|
# Search settings
|
|
MOBILEDE_FILTERED_SEARCH_URL=
|
|
MOBILEDE_FILTERED_SEARCH_URLS=
|
|
MOBILEDE_REQUEST_DELAY_SECONDS=0.30
|
|
MOBILEDE_CONCURRENT_PAGES=1
|
|
MOBILEDE_CURSOR_ENABLED=true
|
|
MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
|
|
MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
|
|
MOBILEDE_RUNTIME_INITIAL_TASKS=3
|
|
MOBILEDE_SEGMENT_PAGE_WINDOW=10
|
|
MOBILEDE_RESULTS_PER_PAGE=20
|
|
MOBILEDE_MAX_PAGE_NUMBER=50
|
|
MOBILEDE_SEGMENT_TARGET_RESULTS=1000
|
|
MOBILEDE_SKIP_EMPTY_WINDOW=true
|
|
MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS=true
|
|
MOBILEDE_COMPACT_SEGMENTS=true
|
|
MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=false
|
|
|
|
# Planner and overflow
|
|
MOBILEDE_PREPLAN_SEGMENT_PROBES=true
|
|
MOBILEDE_PREPLAN_MAX_SEGMENTS=700
|
|
MOBILEDE_PREPLAN_MAX_PROBES=80
|
|
MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
|
|
MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
|
|
MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
|
|
MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true
|
|
MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true
|
|
MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120
|
|
MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
|
|
MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4
|
|
MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
|
|
MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
|
|
|
|
# Existing car refresh
|
|
MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
|
|
MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
|
|
MOBILEDE_LIGHT_REFRESH_EXISTING=true
|
|
MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
|
|
MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false
|
|
MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1
|
|
MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800
|
|
MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45
|
|
MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2
|
|
MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2
|
|
MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true
|
|
|
|
# Proxy
|
|
MOBILEDE_PROXY_SERVER=
|
|
MOBILEDE_PROXY_USERNAME=
|
|
MOBILEDE_PROXY_PASSWORD=
|
|
SOCKS5_PROXY_HOST=
|
|
SOCKS5_PROXY_PORT=1002
|
|
SOCKS5_PROXY_USER=
|
|
SOCKS5_PROXY_PASS=
|
|
|
|
TZ=UTC
|