diff --git a/.env.example b/.env.example index 67a09fe..affe2ed 100644 --- a/.env.example +++ b/.env.example @@ -1,69 +1,104 @@ -# Параметры mobile.de для Docker. +# Docker environment for mobile.de -# Настройки PostgreSQL. +# Database POSTGRES_USER=mobilede POSTGRES_PASSWORD=mobilede POSTGRES_DB=mobilede_scraper MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper +MOBILEDE_DATABASE_ECHO=false +MOBILEDE_DATABASE_POOL_SIZE=20 +MOBILEDE_DATABASE_MAX_OVERFLOW=40 +MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800 -# Порты сервисов. +# Ports MOBILEDE_HOST_API_PORT=8000 MOBILEDE_HOST_POSTGRES_PORT=5432 MOBILEDE_HOST_REDIS_PORT=6379 -# Временные legacy-переменные. -MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper -MOBILEDE_DATABASE_ECHO=false -MOBILEDE_DATABASE_POOL_SIZE=5 -MOBILEDE_DATABASE_MAX_OVERFLOW=5 -MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800 -# Настройки Redis и Celery. +# Redis and Celery MOBILEDE_REDIS_URL=redis://redis:6379/0 CELERY_BROKER_URL=redis://redis:6379/0 CELERY_RESULT_BACKEND=redis://redis:6379/0 CELERY_TASK_SOFT_TIME_LIMIT=86400 CELERY_TASK_TIME_LIMIT=86520 CELERY_BROKER_VISIBILITY_TIMEOUT=90000 -CELERY_WORKER_CONCURRENCY=1 -CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 -IAAI_STARTUP_SYNC_ENABLED=true -MOBILEDE_STARTUP_MAX_PAGES=5 -MOBILEDE_BEAT_MAX_PAGES=5 +CELERY_WORKER_CONCURRENCY=3 +CELERY_WORKER_POOL=prefork +CELERY_WORKER_MAX_TASKS_PER_CHILD=5 +CELERY_BATCH_SIZE=1000 -# Одноразовые CLI-сервисы. -MOBILEDE_SEARCH_START_PAGE=1 -MOBILEDE_SEARCH_MAX_PAGES=1 -MOBILEDE_REQUEST_DELAY_SECONDS=0.7 -MOBILEDE_SYNC_LANE=mobile_de_cars -MOBILEDE_LISTING_ID=449929602 -MOBILEDE_CONTINUOUS_SYNC_ENABLED=true -MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=0 +# Runtime mode +MOBILEDE_STARTUP_SYNC_ENABLED=false +MOBILEDE_BEAT_SYNC_ENABLED=false +MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true +MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600 +MOBILEDE_CONTINUOUS_SYNC_ENABLED=false +MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600 +MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1 + +# Search settings +MOBILEDE_FILTERED_SEARCH_URL= +MOBILEDE_FILTERED_SEARCH_URLS= +MOBILEDE_STARTUP_MAX_PAGES=100 +MOBILEDE_BEAT_MAX_PAGES=100 +MOBILEDE_REQUEST_DELAY_SECONDS=0.30 +MOBILEDE_CONCURRENT_PAGES=1 +MOBILEDE_CURSOR_ENABLED=true MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10 +MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true +MOBILEDE_RUNTIME_INITIAL_TASKS=3 +MOBILEDE_SEGMENT_PAGE_WINDOW=10 +MOBILEDE_RESULTS_PER_PAGE=20 +MOBILEDE_MAX_PAGE_NUMBER=50 +MOBILEDE_SEGMENT_TARGET_RESULTS=1000 +MOBILEDE_SKIP_EMPTY_WINDOW=true +MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS=true +MOBILEDE_COMPACT_SEGMENTS=true +MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=false -# Прокси для HTTP-запросов. -# Пример: -# MOBILEDE_PROXY_SERVER=http://host:port -# MOBILEDE_PROXY_USERNAME=username -# MOBILEDE_PROXY_PASSWORD=password +# Planner and overflow +MOBILEDE_PREPLAN_SEGMENT_PROBES=true +MOBILEDE_PREPLAN_MAX_SEGMENTS=700 +MOBILEDE_PREPLAN_MAX_PROBES=80 +MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2 +MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5 +MOBILEDE_DYNAMIC_SEGMENT_PROBES=false +MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true +MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true +MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120 +MOBILEDE_OVERFLOW_SPLIT_ENABLED=true +MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4 +MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3 +MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000 + +# Existing car refresh +MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false +MOBILEDE_INCREMENTAL_PAGE_WINDOW=10 +MOBILEDE_LIGHT_REFRESH_EXISTING=true +MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1 +MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false +MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1 +MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800 +MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45 +MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2 +MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2 +MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true + +# Proxy MOBILEDE_PROXY_SERVER= MOBILEDE_PROXY_USERNAME= MOBILEDE_PROXY_PASSWORD= - -# Опциональный SOCKS5 bridge. -# При SOCKS5_PROXY_HOST мост поднимается на http://127.0.0.1:8899. -# Тогда MOBILEDE_PROXY_SERVER можно не задавать. SOCKS5_PROXY_HOST= SOCKS5_PROXY_PORT=1002 SOCKS5_PROXY_USER= SOCKS5_PROXY_PASS= -# Логи и runtime. +# IAAI +IAAI_STARTUP_SYNC_ENABLED=true IAAI_LOG_LEVEL=INFO -IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json -TZ=UTC - -# Старые browser-переменные. -IAAI_HEADLESS=true IAAI_BROWSER_ENGINE=chromium IAAI_TOKENS_FILE=/data/tokens.json +IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json IAAI_SELF_HEAL_ENABLED=true + +TZ=UTC diff --git a/.gitignore b/.gitignore index dcb8a5a..cd3ffec 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,9 @@ tokens.json .tmp_db_check.sql deploy-*.tar.gz tmp_worker_log.txt +*.bak.* +_snapshot_info.txt +.DS_Store +Thumbs.db +.idea/ +.ruff_cache/ diff --git a/docker-compose.yml b/docker-compose.yml index e9499dd..d36dd80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,6 @@ services: # База данных. postgres: image: postgres:16-alpine - container_name: mobilede-postgres restart: unless-stopped environment: POSTGRES_USER: ${POSTGRES_USER:-mobilede} @@ -123,7 +122,6 @@ services: # Очередь Redis. redis: image: redis:7-alpine - container_name: mobilede-redis restart: unless-stopped ports: - "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379" @@ -147,7 +145,6 @@ services: # Миграции. migrate: <<: *app-service - container_name: mobilede-migrate restart: "no" depends_on: postgres: @@ -157,7 +154,6 @@ services: # API сервис. api: <<: *app-service - container_name: mobilede-api restart: unless-stopped ports: - "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000" @@ -185,7 +181,6 @@ services: # Worker сервис. worker: <<: *worker-service - container_name: mobilede-worker restart: unless-stopped init: true depends_on: @@ -215,7 +210,6 @@ services: # Beat сервис. beat: <<: *worker-service - container_name: mobilede-beat restart: unless-stopped init: true depends_on: @@ -243,7 +237,6 @@ services: # Одноразовый сбор страниц поиска. mobilede-search: <<: *app-service - container_name: mobilede-search profiles: ["cli"] restart: "no" depends_on: @@ -262,7 +255,6 @@ services: # Одноразовый sync поиска в БД. mobilede-sync-search: <<: *app-service - container_name: mobilede-sync-search profiles: ["cli"] restart: "no" depends_on: @@ -282,7 +274,6 @@ services: # Одноразовый сбор карточки по ID. mobilede-detail: <<: *app-service - container_name: mobilede-detail profiles: ["cli"] restart: "no" depends_on: diff --git a/mobilede_scraper/discovery/__init__.py b/mobilede_scraper/discovery/__init__.py deleted file mode 100644 index c9c2ef6..0000000 --- a/mobilede_scraper/discovery/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__: list[str] = []