fix scraper flow

docker-compose.yml

@@ -1,5 +1,5 @@
 x-app-env: &app-env
-  # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
+  # Всегда используем Postgres.
   IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
   IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
@@ -8,20 +8,28 @@ x-app-env: &app-env
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  # 0 => без лимита (в коде интерпретируется как None).
-  # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
+  # 0 = без лимита.
   CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
-  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
   CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
-  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
+  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
+  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
+  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
   IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
   IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
   IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
   IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
   IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
   IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
   IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
   IAAI_BROWSER_ENGINE: chromium
+  # Self-heal для worker.
+  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
+  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
+  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
+  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
+  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -42,7 +50,7 @@ x-worker-service: &worker-service
     - tokens_data:/data
 
 services:
-  # PostgreSQL
+  # PostgreSQL.
   postgres:
     image: postgres:16-alpine
     container_name: iaai-postgres
@@ -66,7 +74,7 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Redis (Celery broker)
+  # Redis.
   redis:
     image: redis:7-alpine
     container_name: iaai-redis
@@ -90,7 +98,7 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # DB migrations
+  # Миграции.
   migrate:
     <<: *app-service
     container_name: iaai-migrate
@@ -100,7 +108,7 @@ services:
         condition: service_healthy
     command: alembic upgrade head
 
-  # FastAPI
+  # API.
   api:
     <<: *app-service
     container_name: iaai-api
@@ -128,14 +136,11 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Celery Worker
+  # Worker.
   worker:
     <<: *worker-service
-    container_name: iaai-worker
     restart: unless-stopped
     init: true
-    mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
-    memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
     depends_on:
       migrate:
         condition: service_completed_successfully
@@ -146,9 +151,10 @@ services:
       celery -A iaai_scraper.worker.celery_app worker
       --loglevel=info --concurrency=1 --pool=solo
       --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-1}
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
     healthcheck:
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
+      # Проверка worker.
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
       interval: 60s
       timeout: 10s
       retries: 3
@@ -159,7 +165,7 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Celery Beat (периодический планировщик)
+  # Beat.
   beat:
     <<: *worker-service
     container_name: iaai-beat