refactor mobile.de parser, fix country mapping, update README

2026-05-04 18:12:10 +03:00
parent edddfc884c
commit 800867b4e6
46 changed files with 2887 additions and 10900 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,13 +1,11 @@
-x-app-env: &app-env
-  # Legacy env name is still used by Settings; values are mobile.de defaults.
-  IAAI_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
+x-app-env: &app-env
  MOBILEDE_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
-  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
+  MOBILEDE_REDIS_URL: ${MOBILEDE_REDIS_URL:-redis://redis:6379/0}
  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
-  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  IAAI_DATABASE_POOL_SIZE: ${IAAI_DATABASE_POOL_SIZE:-20}
-  IAAI_DATABASE_MAX_OVERFLOW: ${IAAI_DATABASE_MAX_OVERFLOW:-40}
+  MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS: ${MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-20}
+  MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-40}
  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
@@ -21,7 +19,7 @@ x-app-env: &app-env
  IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0}
  IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1}
  IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
-  # 0 = Р±РµР· Р»РёРјРёС‚Р°.
+  # 0 = без лимита.
  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
  CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
@@ -31,6 +29,8 @@ x-app-env: &app-env
  MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100}
  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0}
  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
+  MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-}
+  MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-}
  MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
  MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true}
  MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-3600}
@@ -43,12 +43,21 @@ x-app-env: &app-env
  MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50}
  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1000}
  MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
+  MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-false}
+  MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-1000}
+  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40}
  MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
  MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-false}
  MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
  MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true}
  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1}
+  MOBILEDE_FLARESOLVERR_ENABLED: ${MOBILEDE_FLARESOLVERR_ENABLED:-false}
+  MOBILEDE_FLARESOLVERR_URL: ${MOBILEDE_FLARESOLVERR_URL:-http://flaresolverr:8191/v1}
+  MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS: ${MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS:-120}
+  MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS: ${MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS:-60000}
+  MOBILEDE_FLARESOLVERR_SESSION: ${MOBILEDE_FLARESOLVERR_SESSION:-}
+  MOBILEDE_FLARESOLVERR_STATUSES: ${MOBILEDE_FLARESOLVERR_STATUSES:-403,429,503}
  IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
@@ -65,13 +74,13 @@ x-app-env: &app-env
  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json}
  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
  IAAI_BROWSER_ENGINE: chromium
-  # Self-heal РґР»СЏ worker.
+  # Автовосстановление worker.
  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
-  # Р•СЃР»Рё РІ Postgres РЅРµС‚ Р·Р°РїРёСЃРµР№ РґРѕР»СЊС€Рµ 60 РјРёРЅСѓС‚ РїСЂРё Р°РєС‚РёРІРЅРѕР№ СЂР°Р±РѕС‚Рµ вЂ” РїРѕР»РЅС‹Р№ restart СЃ segment 1.
+  # При простое БД перезапускаем worker.
  IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
  TZ: ${TZ:-UTC}

@@ -93,7 +102,7 @@ x-worker-service: &worker-service
    - tokens_data:/data

 services:
-  # PostgreSQL.
+  # База данных.
  postgres:
    image: postgres:16-alpine
    container_name: mobilede-postgres
@@ -117,7 +126,7 @@ services:
        max-size: "10m"
        max-file: "5"

-  # Redis.
+  # Очередь Redis.
  redis:
    image: redis:7-alpine
    container_name: mobilede-redis
@@ -141,7 +150,24 @@ services:
        max-size: "10m"
        max-file: "5"

-  # РњРёРіСЂР°С†РёРё.
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    container_name: mobilede-flaresolverr
+    restart: unless-stopped
+    environment:
+      LOG_LEVEL: ${FLARESOLVERR_LOG_LEVEL:-info}
+      LOG_HTML: ${FLARESOLVERR_LOG_HTML:-false}
+      CAPTCHA_SOLVER: ${FLARESOLVERR_CAPTCHA_SOLVER:-none}
+      TZ: ${TZ:-UTC}
+    ports:
+      - "127.0.0.1:${MOBILEDE_HOST_FLARESOLVERR_PORT:-8191}:8191"
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
+  # Миграции.
  migrate:
    <<: *app-service
    container_name: mobilede-migrate
@@ -151,7 +177,7 @@ services:
        condition: service_healthy
    command: alembic upgrade head

-  # API.
+  # API сервис.
  api:
    <<: *app-service
    container_name: mobilede-api
@@ -179,7 +205,7 @@ services:
        max-size: "10m"
        max-file: "5"

-  # Worker.
+  # Worker сервис.
  worker:
    <<: *worker-service
    container_name: mobilede-worker
@@ -197,7 +223,7 @@ services:
      --pidfile=/tmp/celery-worker.pid
      -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
    healthcheck:
-      # РџСЂРѕРІРµСЂРєР° worker.
+      # Проверка worker.
      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"]
      interval: 60s
      timeout: 10s
@@ -209,7 +235,7 @@ services:
        max-size: "10m"
        max-file: "5"

-  # Beat.
+  # Beat сервис.
  beat:
    <<: *worker-service
    container_name: mobilede-beat
@@ -237,7 +263,7 @@ services:
        max-size: "10m"
        max-file: "5"

-  # One-shot CLI: collect mobile.de search pages into artifacts/json.
+  # Одноразовый сбор страниц поиска.
  mobilede-search:
    <<: *app-service
    container_name: mobilede-search
@@ -256,7 +282,7 @@ services:
      --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
      --output /app/artifacts/json/mobilede_search.json

-  # One-shot CLI: collect and upsert mobile.de search pages into Postgres.
+  # Одноразовый sync поиска в БД.
  mobilede-sync-search:
    <<: *app-service
    container_name: mobilede-sync-search
@@ -276,7 +302,7 @@ services:
      --lane ${MOBILEDE_SYNC_LANE:-mobile_de_cars}
      --output /app/artifacts/json/mobilede_sync_search.json

-  # One-shot CLI: collect one detail page by listing id.
+  # Одноразовый сбор карточки по ID.
  mobilede-detail:
    <<: *app-service
    container_name: mobilede-detail