a4c93a1df1
improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
67 lines
2.2 KiB
Plaintext
67 lines
2.2 KiB
Plaintext
IAAI_HEADLESS=true
|
|
IAAI_LOG_LEVEL=INFO
|
|
# IAAI_LOG_FILE=iaai_scraper.log
|
|
|
|
# Network capture settings.
|
|
IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
|
|
IAAI_MAX_CAPTURED_REQUESTS=40
|
|
IAAI_MAX_CAPTURED_JSON_RESPONSES=30
|
|
|
|
# Sequential listing-first mode.
|
|
IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
|
|
IAAI_MAX_PAGES_PER_RUN=10
|
|
IAAI_MAX_VEHICLES_PER_RUN=50000
|
|
IAAI_PAGE_LINK_LIMIT=500
|
|
IAAI_INCLUDE_PAGINATION=true
|
|
IAAI_COLLECT_CURRENT_PAGE_ONLY=false
|
|
|
|
# Паузы (отключены для максимальной скорости; включи на VPS если попадаешь под блокировки).
|
|
IAAI_HUMAN_PACE_ENABLED=false
|
|
IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
|
|
IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
|
|
IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
|
|
IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
|
|
IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
|
|
IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
|
|
IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
|
|
IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
|
|
IAAI_BETWEEN_VEHICLES_MIN_S=0.02
|
|
IAAI_BETWEEN_VEHICLES_MAX_S=0.08
|
|
IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
|
|
IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
|
|
|
|
# Sync settings
|
|
IAAI_SYNC_ONLY_NEW=true
|
|
IAAI_TOKENS_FILE=/data/tokens.json
|
|
IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
|
|
|
|
# Retry / backoff
|
|
IAAI_RETRY_DELAY_SECONDS=2.5
|
|
IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
|
|
IAAI_RETRY_JITTER_SECONDS=0.25
|
|
|
|
# Proxy (HTTP/HTTPS preferred for Playwright)
|
|
# Chromium does not support SOCKS5 proxy authentication directly.
|
|
# Use residential or mobile USA proxy.
|
|
# IAAI_PROXY_SERVER=http://proxy.example.com:8080
|
|
# IAAI_PROXY_USERNAME=
|
|
# IAAI_PROXY_PASSWORD=
|
|
|
|
# Database (PostgreSQL).
|
|
IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
|
|
IAAI_DATABASE_ECHO=false
|
|
IAAI_DATABASE_POOL_SIZE=10
|
|
IAAI_DATABASE_MAX_OVERFLOW=20
|
|
|
|
# ─── Redis (Celery broker) ─────────────────────────────────
|
|
IAAI_REDIS_URL=redis://redis:6379/0
|
|
|
|
# ─── Celery ────────────────────────────────────────────────
|
|
CELERY_BROKER_URL=redis://redis:6379/0
|
|
CELERY_RESULT_BACKEND=redis://redis:6379/0
|
|
CELERY_TASK_SOFT_TIME_LIMIT=600
|
|
CELERY_TASK_TIME_LIMIT=900
|
|
CELERY_WORKER_CONCURRENCY=1
|
|
CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
|
|
CELERY_BEAT_SYNC_LIMIT=26
|