Prepare mobile de parser release

This commit is contained in:
qananasikq
2026-04-27 21:10:17 +03:00
parent 31f87a9bdf
commit b86bcd04b8
21 changed files with 2835 additions and 423 deletions

View File

@@ -1,54 +1,20 @@
IAAI_HEADLESS=true
IAAI_LOG_LEVEL=INFO
# mobile.de scraper Docker defaults
IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
IAAI_MAX_CAPTURED_REQUESTS=40
IAAI_MAX_CAPTURED_JSON_RESPONSES=20
# PostgreSQL used by docker-compose.
POSTGRES_USER=mobilede
POSTGRES_PASSWORD=mobilede
POSTGRES_DB=mobilede_scraper
MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
IAAI_FILTERED_SEARCH_URL=
IAAI_FILTERED_SEARCH_URLS=
IAAI_LISTING_SEGMENTS=runtime
IAAI_MAX_PAGES_PER_RUN=999999
IAAI_MAX_VEHICLES_PER_RUN=999999
IAAI_PAGE_LINK_LIMIT=999999
IAAI_INCLUDE_PAGINATION=true
IAAI_COLLECT_CURRENT_PAGE_ONLY=false
IAAI_HUMAN_PACE_ENABLED=true
IAAI_PARALLEL_TABS=10
CELERY_BATCH_SIZE=100
IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
IAAI_BETWEEN_VEHICLES_MIN_S=0.02
IAAI_BETWEEN_VEHICLES_MAX_S=0.08
IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
IAAI_SYNC_ONLY_NEW=false
IAAI_TOKENS_FILE=/data/tokens.json
IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
IAAI_MAX_RETRIES=5
IAAI_RETRY_DELAY_SECONDS=4
IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
IAAI_RETRY_JITTER_SECONDS=0.5
IAAI_TIMEOUT_MS=90000
IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
# Legacy env names still consumed by Settings until the old IAAI core is fully removed.
IAAI_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
IAAI_DATABASE_ECHO=false
IAAI_DATABASE_POOL_SIZE=5
IAAI_DATABASE_MAX_OVERFLOW=5
IAAI_DATABASE_POOL_RECYCLE_SECONDS=1800
# Redis / Celery stack.
IAAI_REDIS_URL=redis://redis:6379/0
CELERY_BROKER_URL=redis://redis:6379/0
CELERY_RESULT_BACKEND=redis://redis:6379/0
CELERY_TASK_SOFT_TIME_LIMIT=86400
@@ -56,7 +22,24 @@ CELERY_TASK_TIME_LIMIT=86520
CELERY_BROKER_VISIBILITY_TIMEOUT=90000
CELERY_WORKER_CONCURRENCY=1
CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
CELERY_BEAT_SYNC_LIMIT=0
IAAI_ALWAYS_FULL_SCAN=true
SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600
IAAI_STARTUP_SYNC_ENABLED=true
MOBILEDE_STARTUP_MAX_PAGES=5
MOBILEDE_BEAT_MAX_PAGES=5
# mobile.de one-shot CLI services.
MOBILEDE_SEARCH_START_PAGE=1
MOBILEDE_SEARCH_MAX_PAGES=1
MOBILEDE_REQUEST_DELAY_SECONDS=0.7
MOBILEDE_SYNC_LANE=mobile_de_cars
MOBILEDE_LISTING_ID=449929602
# Logging / runtime.
IAAI_LOG_LEVEL=INFO
IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
TZ=UTC
# Legacy browser settings kept for old deprecated IAAI commands only.
IAAI_HEADLESS=true
IAAI_BROWSER_ENGINE=chromium
IAAI_TOKENS_FILE=/data/tokens.json
IAAI_SELF_HEAL_ENABLED=true