Prepare mobile de parser release

2026-04-27 21:10:17 +03:00
parent 31f87a9bdf
commit b86bcd04b8
21 changed files with 2835 additions and 423 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,6 +1,7 @@
 x-app-env: &app-env
-  # Всегда используем Postgres.
-  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+  # Legacy env name is still used by Settings; values are mobile.de defaults.
+  IAAI_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
+  MOBILEDE_DATABASE_URL: ${MOBILEDE_DATABASE_URL:-postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper}
  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
@@ -20,12 +21,32 @@ x-app-env: &app-env
  IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0}
  IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1}
  IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
-  # 0 = без лимита.
+  # 0 = Р±РµР· Р»РёРјРёС‚Р°.
  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
  CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000}
+  MOBILEDE_STARTUP_MAX_PAGES: ${MOBILEDE_STARTUP_MAX_PAGES:-100}
+  MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100}
+  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0}
+  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
+  MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
+  MOBILEDE_CONTINUOUS_SYNC_ENABLED: ${MOBILEDE_CONTINUOUS_SYNC_ENABLED:-true}
+  MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS: ${MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS:-0}
+  MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-100}
+  MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
+  MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
+  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2}
+  MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10}
+  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1800}
+  MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
+  MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
+  MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
+  MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
+  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true}
+  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1}
+  IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
@@ -41,13 +62,13 @@ x-app-env: &app-env
  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json}
  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
  IAAI_BROWSER_ENGINE: chromium
-  # Self-heal для worker.
+  # Self-heal РґР»СЏ worker.
  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
-  # Если в Postgres нет записей дольше 60 минут при активной работе — полный restart с segment 1.
+  # Р•СЃР»Рё РІ Postgres РЅРµС‚ Р·Р°РїРёСЃРµР№ РґРѕР»СЊС€Рµ 60 РјРёРЅСѓС‚ РїСЂРё Р°РєС‚РёРІРЅРѕР№ СЂР°Р±РѕС‚Рµ вЂ” РїРѕР»РЅС‹Р№ restart СЃ segment 1.
  IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
  TZ: ${TZ:-UTC}

@@ -72,18 +93,18 @@ services:
  # PostgreSQL.
  postgres:
    image: postgres:16-alpine
-    container_name: iaai-postgres
+    container_name: mobilede-postgres
    restart: unless-stopped
    environment:
-      POSTGRES_USER: iaai
-      POSTGRES_PASSWORD: iaai
-      POSTGRES_DB: iaai_scraper
+      POSTGRES_USER: ${POSTGRES_USER:-mobilede}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mobilede}
+      POSTGRES_DB: ${POSTGRES_DB:-mobilede_scraper}
    ports:
      - "127.0.0.1:5432:5432"
    volumes:
      - pgdata:/var/lib/postgresql/data
    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mobilede} -d ${POSTGRES_DB:-mobilede_scraper}"]
      interval: 5s
      timeout: 3s
      retries: 5
@@ -96,7 +117,7 @@ services:
  # Redis.
  redis:
    image: redis:7-alpine
-    container_name: iaai-redis
+    container_name: mobilede-redis
    restart: unless-stopped
    ports:
      - "127.0.0.1:6379:6379"
@@ -117,10 +138,10 @@ services:
        max-size: "10m"
        max-file: "5"

-  # Миграции.
+  # РњРёРіСЂР°С†РёРё.
  migrate:
    <<: *app-service
-    container_name: iaai-migrate
+    container_name: mobilede-migrate
    restart: "no"
    depends_on:
      postgres:
@@ -130,7 +151,7 @@ services:
  # API.
  api:
    <<: *app-service
-    container_name: iaai-api
+    container_name: mobilede-api
    restart: unless-stopped
    ports:
      - "127.0.0.1:8000:8000"
@@ -158,6 +179,7 @@ services:
  # Worker.
  worker:
    <<: *worker-service
+    container_name: mobilede-worker
    restart: unless-stopped
    init: true
    depends_on:
@@ -170,9 +192,9 @@ services:
      celery -A iaai_scraper.worker.celery_app worker
      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork}
      --pidfile=/tmp/celery-worker.pid
-      -Q iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+      -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
    healthcheck:
-      # Проверка worker.
+      # РџСЂРѕРІРµСЂРєР° worker.
      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
      interval: 60s
      timeout: 10s
@@ -187,7 +209,7 @@ services:
  # Beat.
  beat:
    <<: *worker-service
-    container_name: iaai-beat
+    container_name: mobilede-beat
    restart: unless-stopped
    init: true
    depends_on:
@@ -212,6 +234,61 @@ services:
        max-size: "10m"
        max-file: "5"

+  # One-shot CLI: collect mobile.de search pages into artifacts/json.
+  mobilede-search:
+    <<: *app-service
+    container_name: mobilede-search
+    profiles: ["cli"]
+    restart: "no"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    volumes:
+      - ./runtime_config.json:/app/runtime_config.json:ro
+      - ./artifacts:/app/artifacts
+    command: >
+      mobilede search
+      --max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1}
+      --page ${MOBILEDE_SEARCH_START_PAGE:-1}
+      --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
+      --output /app/artifacts/json/mobilede_search.json
+
+  # One-shot CLI: collect and upsert mobile.de search pages into Postgres.
+  mobilede-sync-search:
+    <<: *app-service
+    container_name: mobilede-sync-search
+    profiles: ["cli"]
+    restart: "no"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    volumes:
+      - ./runtime_config.json:/app/runtime_config.json:ro
+      - ./artifacts:/app/artifacts
+    command: >
+      mobilede sync-search
+      --max-pages ${MOBILEDE_SEARCH_MAX_PAGES:-1}
+      --page ${MOBILEDE_SEARCH_START_PAGE:-1}
+      --delay ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.7}
+      --lane ${MOBILEDE_SYNC_LANE:-mobile_de_cars}
+      --output /app/artifacts/json/mobilede_sync_search.json
+
+  # One-shot CLI: collect one detail page by listing id.
+  mobilede-detail:
+    <<: *app-service
+    container_name: mobilede-detail
+    profiles: ["cli"]
+    restart: "no"
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    volumes:
+      - ./runtime_config.json:/app/runtime_config.json:ro
+      - ./artifacts:/app/artifacts
+    command: >
+      mobilede detail ${MOBILEDE_LISTING_ID:-449929602}
+      --output /app/artifacts/json/mobilede_detail.json
+
 volumes:
  pgdata:
  redisdata: