Update setup

2026-05-06 21:07:24 +03:00
parent 519686c0b7
commit 5049a29ef7
4 changed files with 79 additions and 48 deletions
--- a/.env.example
+++ b/.env.example
@@ -1,69 +1,104 @@
-# Параметры mobile.de для Docker.
+# Docker environment for mobile.de

-# Настройки PostgreSQL.
+# Database
 POSTGRES_USER=mobilede
 POSTGRES_PASSWORD=mobilede
 POSTGRES_DB=mobilede_scraper
 MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
+MOBILEDE_DATABASE_ECHO=false
+MOBILEDE_DATABASE_POOL_SIZE=20
+MOBILEDE_DATABASE_MAX_OVERFLOW=40
+MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800

-# Порты сервисов.
+# Ports
 MOBILEDE_HOST_API_PORT=8000
 MOBILEDE_HOST_POSTGRES_PORT=5432
 MOBILEDE_HOST_REDIS_PORT=6379
-# Временные legacy-переменные.
-MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
-MOBILEDE_DATABASE_ECHO=false
-MOBILEDE_DATABASE_POOL_SIZE=5
-MOBILEDE_DATABASE_MAX_OVERFLOW=5
-MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800

-# Настройки Redis и Celery.
+# Redis and Celery
 MOBILEDE_REDIS_URL=redis://redis:6379/0
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
 CELERY_TASK_SOFT_TIME_LIMIT=86400
 CELERY_TASK_TIME_LIMIT=86520
 CELERY_BROKER_VISIBILITY_TIMEOUT=90000
-CELERY_WORKER_CONCURRENCY=1
-CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
-IAAI_STARTUP_SYNC_ENABLED=true
-MOBILEDE_STARTUP_MAX_PAGES=5
-MOBILEDE_BEAT_MAX_PAGES=5
+CELERY_WORKER_CONCURRENCY=3
+CELERY_WORKER_POOL=prefork
+CELERY_WORKER_MAX_TASKS_PER_CHILD=5
+CELERY_BATCH_SIZE=1000

-# Одноразовые CLI-сервисы.
-MOBILEDE_SEARCH_START_PAGE=1
-MOBILEDE_SEARCH_MAX_PAGES=1
-MOBILEDE_REQUEST_DELAY_SECONDS=0.7
-MOBILEDE_SYNC_LANE=mobile_de_cars
-MOBILEDE_LISTING_ID=449929602
-MOBILEDE_CONTINUOUS_SYNC_ENABLED=true
-MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=0
+# Runtime mode
+MOBILEDE_STARTUP_SYNC_ENABLED=false
+MOBILEDE_BEAT_SYNC_ENABLED=false
+MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
+MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
+MOBILEDE_CONTINUOUS_SYNC_ENABLED=false
+MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
+MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
+
+# Search settings
+MOBILEDE_FILTERED_SEARCH_URL=
+MOBILEDE_FILTERED_SEARCH_URLS=
+MOBILEDE_STARTUP_MAX_PAGES=100
+MOBILEDE_BEAT_MAX_PAGES=100
+MOBILEDE_REQUEST_DELAY_SECONDS=0.30
+MOBILEDE_CONCURRENT_PAGES=1
+MOBILEDE_CURSOR_ENABLED=true
 MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
+MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
+MOBILEDE_RUNTIME_INITIAL_TASKS=3
+MOBILEDE_SEGMENT_PAGE_WINDOW=10
+MOBILEDE_RESULTS_PER_PAGE=20
+MOBILEDE_MAX_PAGE_NUMBER=50
+MOBILEDE_SEGMENT_TARGET_RESULTS=1000
+MOBILEDE_SKIP_EMPTY_WINDOW=true
+MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS=true
+MOBILEDE_COMPACT_SEGMENTS=true
+MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=false

-# Прокси для HTTP-запросов.
-# Пример:
-# MOBILEDE_PROXY_SERVER=http://host:port
-# MOBILEDE_PROXY_USERNAME=username
-# MOBILEDE_PROXY_PASSWORD=password
+# Planner and overflow
+MOBILEDE_PREPLAN_SEGMENT_PROBES=true
+MOBILEDE_PREPLAN_MAX_SEGMENTS=700
+MOBILEDE_PREPLAN_MAX_PROBES=80
+MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
+MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
+MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
+MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true
+MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true
+MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120
+MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
+MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4
+MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
+MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
+
+# Existing car refresh
+MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
+MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
+MOBILEDE_LIGHT_REFRESH_EXISTING=true
+MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
+MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false
+MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1
+MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800
+MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45
+MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2
+MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2
+MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true
+
+# Proxy
 MOBILEDE_PROXY_SERVER=
 MOBILEDE_PROXY_USERNAME=
 MOBILEDE_PROXY_PASSWORD=
-
-# Опциональный SOCKS5 bridge.
-# При SOCKS5_PROXY_HOST мост поднимается на http://127.0.0.1:8899.
-# Тогда MOBILEDE_PROXY_SERVER можно не задавать.
 SOCKS5_PROXY_HOST=
 SOCKS5_PROXY_PORT=1002
 SOCKS5_PROXY_USER=
 SOCKS5_PROXY_PASS=

-# Логи и runtime.
+# IAAI
+IAAI_STARTUP_SYNC_ENABLED=true
 IAAI_LOG_LEVEL=INFO
-IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
-TZ=UTC
-
-# Старые browser-переменные.
-IAAI_HEADLESS=true
 IAAI_BROWSER_ENGINE=chromium
 IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 IAAI_SELF_HEAL_ENABLED=true
+
+TZ=UTC
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,9 @@ tokens.json
 .tmp_db_check.sql
 deploy-*.tar.gz
 tmp_worker_log.txt
+*.bak.*
+_snapshot_info.txt
+.DS_Store
+Thumbs.db
+.idea/
+.ruff_cache/
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -99,7 +99,6 @@ services:
  # База данных.
  postgres:
    image: postgres:16-alpine
-    container_name: mobilede-postgres
    restart: unless-stopped
    environment:
      POSTGRES_USER: ${POSTGRES_USER:-mobilede}
@@ -123,7 +122,6 @@ services:
  # Очередь Redis.
  redis:
    image: redis:7-alpine
-    container_name: mobilede-redis
    restart: unless-stopped
    ports:
      - "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379"
@@ -147,7 +145,6 @@ services:
  # Миграции.
  migrate:
    <<: *app-service
-    container_name: mobilede-migrate
    restart: "no"
    depends_on:
      postgres:
@@ -157,7 +154,6 @@ services:
  # API сервис.
  api:
    <<: *app-service
-    container_name: mobilede-api
    restart: unless-stopped
    ports:
      - "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000"
@@ -185,7 +181,6 @@ services:
  # Worker сервис.
  worker:
    <<: *worker-service
-    container_name: mobilede-worker
    restart: unless-stopped
    init: true
    depends_on:
@@ -215,7 +210,6 @@ services:
  # Beat сервис.
  beat:
    <<: *worker-service
-    container_name: mobilede-beat
    restart: unless-stopped
    init: true
    depends_on:
@@ -243,7 +237,6 @@ services:
  # Одноразовый сбор страниц поиска.
  mobilede-search:
    <<: *app-service
-    container_name: mobilede-search
    profiles: ["cli"]
    restart: "no"
    depends_on:
@@ -262,7 +255,6 @@ services:
  # Одноразовый sync поиска в БД.
  mobilede-sync-search:
    <<: *app-service
-    container_name: mobilede-sync-search
    profiles: ["cli"]
    restart: "no"
    depends_on:
@@ -282,7 +274,6 @@ services:
  # Одноразовый сбор карточки по ID.
  mobilede-detail:
    <<: *app-service
-    container_name: mobilede-detail
    profiles: ["cli"]
    restart: "no"
    depends_on:
--- a/mobilede_scraper/discovery/init.py
+++ b/mobilede_scraper/discovery/init.py
@@ -1 +0,0 @@
-__all__: list[str] = []