add encar scraper

This commit is contained in:
qananasikq
2026-04-16 18:01:05 +03:00
parent 5624c1973a
commit 4096d29a07
28 changed files with 3472 additions and 150 deletions

View File

@@ -1,66 +1,28 @@
IAAI_HEADLESS=true
IAAI_LOG_LEVEL=INFO
# IAAI_LOG_FILE=iaai_scraper.log
# ===== Encar Scraper Config =====
# Network capture settings.
IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
IAAI_MAX_CAPTURED_REQUESTS=40
IAAI_MAX_CAPTURED_JSON_RESPONSES=30
# Логирование
ENCAR_LOG_LEVEL=INFO
# Sequential listing-first mode.
IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
IAAI_MAX_PAGES_PER_RUN=10
IAAI_MAX_VEHICLES_PER_RUN=50000
IAAI_PAGE_LINK_LIMIT=500
IAAI_INCLUDE_PAGINATION=true
IAAI_COLLECT_CURRENT_PAGE_ONLY=false
# База данных PostgreSQL
ENCAR_DATABASE_URL=postgresql+psycopg2://encar:encar@postgres:5432/encar_db
ENCAR_DATABASE_ECHO=false
ENCAR_DATABASE_POOL_SIZE=5
ENCAR_DATABASE_MAX_OVERFLOW=10
ENCAR_DATABASE_POOL_RECYCLE_SECONDS=1800
# Паузы (отключены для максимальной скорости; включи на VPS если попадаешь под блокировки).
IAAI_HUMAN_PACE_ENABLED=false
IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
IAAI_BETWEEN_VEHICLES_MIN_S=0.02
IAAI_BETWEEN_VEHICLES_MAX_S=0.08
IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
# Redis
ENCAR_REDIS_URL=redis://redis:6379/1
# Sync settings
IAAI_SYNC_ONLY_NEW=true
IAAI_TOKENS_FILE=/data/tokens.json
IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
# Celery
CELERY_BROKER_URL=redis://redis:6379/1
CELERY_RESULT_BACKEND=redis://redis:6379/1
CELERY_TASK_SOFT_TIME_LIMIT=14400
CELERY_TASK_TIME_LIMIT=14520
CELERY_BROKER_VISIBILITY_TIMEOUT=15000
CELERY_WORKER_CONCURRENCY=4
CELERY_WORKER_MAX_TASKS_PER_CHILD=50
# Retry / backoff
IAAI_RETRY_DELAY_SECONDS=2.5
IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
IAAI_RETRY_JITTER_SECONDS=0.25
# Proxy (HTTP/HTTPS preferred for Playwright)
# Chromium does not support SOCKS5 proxy authentication directly.
# Use residential or mobile USA proxy.
# IAAI_PROXY_SERVER=http://proxy.example.com:8080
# IAAI_PROXY_USERNAME=
# IAAI_PROXY_PASSWORD=
# Database (PostgreSQL).
IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
IAAI_DATABASE_ECHO=false
IAAI_DATABASE_POOL_SIZE=10
IAAI_DATABASE_MAX_OVERFLOW=20
# ─── Redis (Celery broker) ─────────────────────────────────
IAAI_REDIS_URL=redis://redis:6379/0
# ─── Celery ────────────────────────────────────────────────
CELERY_BROKER_URL=redis://redis:6379/0
CELERY_RESULT_BACKEND=redis://redis:6379/0
CELERY_TASK_SOFT_TIME_LIMIT=600
CELERY_TASK_TIME_LIMIT=900
CELERY_WORKER_CONCURRENCY=1
CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
CELERY_BEAT_SYNC_LIMIT=26
# Encar Beat — полный прогон каждые 60 минут
ENCAR_BEAT_INTERVAL_MINUTES=60
# 0 = без лимита, парсит ВСЕ авто
ENCAR_BEAT_LIMIT=0