Update setup

.env.example

@@ -1,69 +1,104 @@
-# Параметры mobile.de для Docker.
+# Docker environment for mobile.de
 
-# Настройки PostgreSQL.
+# Database
 POSTGRES_USER=mobilede
 POSTGRES_PASSWORD=mobilede
 POSTGRES_DB=mobilede_scraper
 MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
+MOBILEDE_DATABASE_ECHO=false
+MOBILEDE_DATABASE_POOL_SIZE=20
+MOBILEDE_DATABASE_MAX_OVERFLOW=40
+MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
 
-# Порты сервисов.
+# Ports
 MOBILEDE_HOST_API_PORT=8000
 MOBILEDE_HOST_POSTGRES_PORT=5432
 MOBILEDE_HOST_REDIS_PORT=6379
-# Временные legacy-переменные.
-MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
-MOBILEDE_DATABASE_ECHO=false
-MOBILEDE_DATABASE_POOL_SIZE=5
-MOBILEDE_DATABASE_MAX_OVERFLOW=5
-MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
 
-# Настройки Redis и Celery.
+# Redis and Celery
 MOBILEDE_REDIS_URL=redis://redis:6379/0
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
 CELERY_TASK_SOFT_TIME_LIMIT=86400
 CELERY_TASK_TIME_LIMIT=86520
 CELERY_BROKER_VISIBILITY_TIMEOUT=90000
-CELERY_WORKER_CONCURRENCY=1
+CELERY_WORKER_CONCURRENCY=3
-CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_WORKER_POOL=prefork
-IAAI_STARTUP_SYNC_ENABLED=true
+CELERY_WORKER_MAX_TASKS_PER_CHILD=5
-MOBILEDE_STARTUP_MAX_PAGES=5
+CELERY_BATCH_SIZE=1000
-MOBILEDE_BEAT_MAX_PAGES=5
 
-# Одноразовые CLI-сервисы.
+# Runtime mode
-MOBILEDE_SEARCH_START_PAGE=1
+MOBILEDE_STARTUP_SYNC_ENABLED=false
-MOBILEDE_SEARCH_MAX_PAGES=1
+MOBILEDE_BEAT_SYNC_ENABLED=false
-MOBILEDE_REQUEST_DELAY_SECONDS=0.7
+MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
-MOBILEDE_SYNC_LANE=mobile_de_cars
+MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
-MOBILEDE_LISTING_ID=449929602
+MOBILEDE_CONTINUOUS_SYNC_ENABLED=false
-MOBILEDE_CONTINUOUS_SYNC_ENABLED=true
+MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
-MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=0
+MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
+
+# Search settings
+MOBILEDE_FILTERED_SEARCH_URL=
+MOBILEDE_FILTERED_SEARCH_URLS=
+MOBILEDE_STARTUP_MAX_PAGES=100
+MOBILEDE_BEAT_MAX_PAGES=100
+MOBILEDE_REQUEST_DELAY_SECONDS=0.30
+MOBILEDE_CONCURRENT_PAGES=1
+MOBILEDE_CURSOR_ENABLED=true
 MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
+MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
+MOBILEDE_RUNTIME_INITIAL_TASKS=3
+MOBILEDE_SEGMENT_PAGE_WINDOW=10
+MOBILEDE_RESULTS_PER_PAGE=20
+MOBILEDE_MAX_PAGE_NUMBER=50
+MOBILEDE_SEGMENT_TARGET_RESULTS=1000
+MOBILEDE_SKIP_EMPTY_WINDOW=true
+MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS=true
+MOBILEDE_COMPACT_SEGMENTS=true
+MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=false
 
-# Прокси для HTTP-запросов.
+# Planner and overflow
-# Пример:
+MOBILEDE_PREPLAN_SEGMENT_PROBES=true
-# MOBILEDE_PROXY_SERVER=http://host:port
+MOBILEDE_PREPLAN_MAX_SEGMENTS=700
-# MOBILEDE_PROXY_USERNAME=username
+MOBILEDE_PREPLAN_MAX_PROBES=80
-# MOBILEDE_PROXY_PASSWORD=password
+MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
+MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
+MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
+MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true
+MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true
+MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120
+MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
+MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4
+MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
+MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
+
+# Existing car refresh
+MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
+MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
+MOBILEDE_LIGHT_REFRESH_EXISTING=true
+MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
+MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false
+MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1
+MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800
+MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45
+MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2
+MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2
+MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true
+
+# Proxy
 MOBILEDE_PROXY_SERVER=
 MOBILEDE_PROXY_USERNAME=
 MOBILEDE_PROXY_PASSWORD=
-
-# Опциональный SOCKS5 bridge.
-# При SOCKS5_PROXY_HOST мост поднимается на http://127.0.0.1:8899.
-# Тогда MOBILEDE_PROXY_SERVER можно не задавать.
 SOCKS5_PROXY_HOST=
 SOCKS5_PROXY_PORT=1002
 SOCKS5_PROXY_USER=
 SOCKS5_PROXY_PASS=
 
-# Логи и runtime.
+# IAAI
+IAAI_STARTUP_SYNC_ENABLED=true
 IAAI_LOG_LEVEL=INFO
-IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
-TZ=UTC
-
-# Старые browser-переменные.
-IAAI_HEADLESS=true
 IAAI_BROWSER_ENGINE=chromium
 IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 IAAI_SELF_HEAL_ENABLED=true
+
+TZ=UTC

.gitignore

@@ -24,3 +24,9 @@ tokens.json
 .tmp_db_check.sql
 deploy-*.tar.gz
 tmp_worker_log.txt
+*.bak.*
+_snapshot_info.txt
+.DS_Store
+Thumbs.db
+.idea/
+.ruff_cache/

docker-compose.yml

@@ -99,7 +99,6 @@ services:
   # База данных.
   postgres:
     image: postgres:16-alpine
-    container_name: mobilede-postgres
     restart: unless-stopped
     environment:
       POSTGRES_USER: ${POSTGRES_USER:-mobilede}
@@ -123,7 +122,6 @@ services:
   # Очередь Redis.
   redis:
     image: redis:7-alpine
-    container_name: mobilede-redis
     restart: unless-stopped
     ports:
       - "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379"
@@ -147,7 +145,6 @@ services:
   # Миграции.
   migrate:
     <<: *app-service
-    container_name: mobilede-migrate
     restart: "no"
     depends_on:
       postgres:
@@ -157,7 +154,6 @@ services:
   # API сервис.
   api:
     <<: *app-service
-    container_name: mobilede-api
     restart: unless-stopped
     ports:
       - "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000"
@@ -185,7 +181,6 @@ services:
   # Worker сервис.
   worker:
     <<: *worker-service
-    container_name: mobilede-worker
     restart: unless-stopped
     init: true
     depends_on:
@@ -215,7 +210,6 @@ services:
   # Beat сервис.
   beat:
     <<: *worker-service
-    container_name: mobilede-beat
     restart: unless-stopped
     init: true
     depends_on:
@@ -243,7 +237,6 @@ services:
   # Одноразовый сбор страниц поиска.
   mobilede-search:
     <<: *app-service
-    container_name: mobilede-search
     profiles: ["cli"]
     restart: "no"
     depends_on:
@@ -262,7 +255,6 @@ services:
   # Одноразовый sync поиска в БД.
   mobilede-sync-search:
     <<: *app-service
-    container_name: mobilede-sync-search
     profiles: ["cli"]
     restart: "no"
     depends_on:
@@ -282,7 +274,6 @@ services:
   # Одноразовый сбор карточки по ID.
   mobilede-detail:
     <<: *app-service
-    container_name: mobilede-detail
     profiles: ["cli"]
     restart: "no"
     depends_on:

mobilede_scraper/discovery/__init__.py

@@ -1 +0,0 @@
-__all__: list[str] = []