fix self heal restart

- docker-compose: switch worker to --pool=solo --concurrency=1 --max-tasks-per-child=1
- tasks.py: remove ThreadPoolExecutor wrapper from _run_browser_job (greenlet crash)
- scraper.py: browser fallback runs sequentially instead of ThreadPoolExecutor
- scraper.py: add gc.collect() after scraper close to prevent memory leaks

.env.example

@@ -55,3 +55,6 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=1
 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
 CELERY_BEAT_SYNC_LIMIT=0
+IAAI_ALWAYS_FULL_SCAN=true
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600

.gitignore

@@ -20,11 +20,3 @@ artifacts/
 celerybeat-schedule*
 tokens_data/
 tokens.json
-
-# Local deployment/debug artifacts
-/*.tgz
-/*.tar.gz
-/*.zip
-/NOW()
-/_speed_check.sql
-/.env.bak*

Dockerfile

@@ -13,12 +13,8 @@ RUN pip install --no-cache-dir uv \
     && pip install --no-cache-dir -r /tmp/requirements.txt \
     && pip install --no-cache-dir playwright-stealth
 
-# Speed-up libs: orjson (fast JSON parse), brotli (HTTP br decompress).
-# Pinned outside uv.lock to avoid lock-regen churn.
-RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0"
-
-# Install both Chromium and Firefox. Chromium is the default engine in Docker
-# because it works more reliably with the current IAAI listing page.
+# Ставим Chromium и Firefox.
+# По умолчанию используем Chromium.
 RUN python -m playwright install chromium firefox
 
 COPY . .
@@ -31,6 +27,6 @@ STOPSIGNAL SIGINT
 
 USER app
 
-# По умолчанию  API, но worker/beat переопределяют CMD в docker-compose
+# По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
 CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -112,32 +112,7 @@ docker compose up -d
 
 `entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
 
-По умолчанию `beat` запускает синхронизацию **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
-
-Текущая стратегия устойчива к багам пагинации IAAI:
-
-- **первый запуск** делает полный bootstrap через `sitemap`;
-- если bootstrap не завершился, автоматически продолжается с чекпоинта;
-- **после первого полного прохода** каждый час выполняется проход по `sitemap`.
-  По умолчанию включён режим **rolling refresh**:
-  - берутся все URL из sitemap,
-  - новые URL добавляются сразу,
-  - исчезнувшие URL помечаются как `is_sold=true`,
-  - уже записанные авто обновляются **батчами по кругу**, а не все разом.
-
-  Это даёт более стабильную нагрузку и снижает риск не уложиться в час.
-
-  Доступные режимы:
-  - `IAAI_HOURLY_MODE=rolling_refresh` — рекомендуемый продовый режим;
-  - `IAAI_HOURLY_MODE=full_refresh` — перепарсить все активные авто за один hourly цикл;
-  - `IAAI_HOURLY_MODE=diff` — только новые авто + sold.
-
-За счёт этого система:
-
-- не зависит от page 39 / 70 / 100,
-- не уходит в бесконечный цикл пагинации,
-- даёт стабильный hourly refresh уже записанных авто без монолитного полного hourly прохода,
-- остаётся быстрой и устойчивой при hourly sync.
+По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
 
 Swagger-документация: `http://localhost:8000/docs`
 
@@ -149,9 +124,10 @@ docker compose ps
 - `worker` имеет healthcheck через `celery inspect ping`
 - `beat` имеет healthcheck по файлу `celerybeat-schedule`
 
+
 ## API
 
-**Здоровье и статистика:**
+**Статистика:**
 - `GET /health` — статус сервиса и подключения к БД
 - `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
 
@@ -267,11 +243,6 @@ pytest -q
 
 ## `pyproject.toml` + `uv.lock`
 
-Проект переведён на современную схему зависимостей:
-
-- `pyproject.toml` — декларация зависимостей и метаданных проекта
-- `uv.lock` — зафиксированные версии для воспроизводимых установок
-
 Локально можно использовать:
 
 ```bash
@@ -279,10 +250,6 @@ uv sync
 uv run pytest -q
 ```
 
-## Runtime-фильтры
-
-Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
-
 ### Секция `sync`
 
 Поддерживаются поля:

alembic/versions/004_add_ingestion_tables.py

@@ -1,8 +1,7 @@
-# Добавление таблиц для новой архитектуры ingestion: candidates, raw snapshots, parse results, retry queue
+# Placeholder migration (ingestion tables not yet needed)
 from typing import Sequence, Union
 
 from alembic import op
-import sqlalchemy as sa
 
 revision: str = "004_add_ingestion_tables"
 down_revision: Union[str, None] = "003_add_composite_indexes"
@@ -11,71 +10,8 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Таблица vehicle_candidates
-    op.create_table(
-        "vehicle_candidates",
-        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
-        sa.Column("url", sa.String(), nullable=False, unique=True, index=True),
-        sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True),
-        sa.Column("status", sa.String(20), nullable=False, server_default="pending", index=True),
-        sa.Column("priority", sa.Integer(), nullable=False, server_default="0"),
-        sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
-    )
-
-    # Индексы для vehicle_candidates
-    op.create_index("ix_vehicle_candidates_url", "vehicle_candidates", ["url"])
-    op.create_index("ix_vehicle_candidates_status_discovered", "vehicle_candidates", ["status", "discovered_at"])
-
-    # Таблица vehicle_raw_snapshots
-    op.create_table(
-        "vehicle_raw_snapshots",
-        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
-        sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True),
-        sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True),
-        sa.Column("method", sa.String(20), nullable=False),
-        sa.Column("success", sa.Boolean(), nullable=False),
-        sa.Column("raw_data", sa.Text(), nullable=True),
-        sa.Column("error_message", sa.Text(), nullable=True),
-    )
-
-    # Индексы для vehicle_raw_snapshots
-    op.create_index("ix_vehicle_raw_snapshots_candidate_id", "vehicle_raw_snapshots", ["candidate_id"])
-    op.create_index("ix_vehicle_raw_snapshots_captured_at", "vehicle_raw_snapshots", ["captured_at"])
-
-    # Таблица vehicle_parse_results
-    op.create_table(
-        "vehicle_parse_results",
-        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
-        sa.Column("snapshot_id", sa.Integer(), sa.ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True),
-        sa.Column("parsed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True),
-        sa.Column("success", sa.Boolean(), nullable=False),
-        sa.Column("parsed_data", sa.Text(), nullable=True),
-        sa.Column("error_message", sa.Text(), nullable=True),
-    )
-
-    # Индексы для vehicle_parse_results
-    op.create_index("ix_vehicle_parse_results_snapshot_id", "vehicle_parse_results", ["snapshot_id"])
-    op.create_index("ix_vehicle_parse_results_parsed_at", "vehicle_parse_results", ["parsed_at"])
-
-    # Таблица vehicle_retry_queue
-    op.create_table(
-        "vehicle_retry_queue",
-        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
-        sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True),
-        sa.Column("reason", sa.String(50), nullable=False),
-        sa.Column("retry_at", sa.DateTime(timezone=True), nullable=False, index=True),
-        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
-        sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
-    )
-
-    # Индексы для vehicle_retry_queue
-    op.create_index("ix_vehicle_retry_queue_candidate_id", "vehicle_retry_queue", ["candidate_id"])
-    op.create_index("ix_vehicle_retry_queue_retry_at", "vehicle_retry_queue", ["retry_at"])
+    pass
 
 
 def downgrade() -> None:
-    op.drop_table("vehicle_retry_queue")
-    op.drop_table("vehicle_parse_results")
-    op.drop_table("vehicle_raw_snapshots")
-    op.drop_table("vehicle_candidates")
+    pass

deploy_vps.sh

@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-APP_DIR="/opt/iaai_scraper_project"
-REPO_URL="${1:-}"
-
-if [[ -z "${REPO_URL}" ]]; then
-  echo "Usage: ./deploy_vps.sh <git-repo-url>"
-  exit 1
-fi
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get update
-apt-get install -y --no-install-recommends \
-  ca-certificates \
-  curl \
-  git \
-  gnupg \
-  lsb-release
-
-install -m 0755 -d /etc/apt/keyrings
-if [[ ! -f /etc/apt/keyrings/docker.gpg ]]; then
-  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-fi
-
-chmod a+r /etc/apt/keyrings/docker.gpg
-ARCH="$(dpkg --print-architecture)"
-CODENAME="$(. /etc/os-release && echo "$VERSION_CODENAME")"
-echo \
-  "deb [arch=${ARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${CODENAME} stable" \
-  > /etc/apt/sources.list.d/docker.list
-
-apt-get update
-apt-get install -y --no-install-recommends \
-  docker-ce \
-  docker-ce-cli \
-  containerd.io \
-  docker-buildx-plugin \
-  docker-compose-plugin
-
-mkdir -p /opt
-
-if [[ -d "${APP_DIR}/.git" ]]; then
-  git -C "${APP_DIR}" fetch --all --prune
-  git -C "${APP_DIR}" reset --hard origin/HEAD
-else
-  rm -rf "${APP_DIR}"
-  git clone "${REPO_URL}" "${APP_DIR}"
-fi
-
-cd "${APP_DIR}"
-
-if [[ ! -f .env ]]; then
-  cp .env.vps.example .env
-fi
-
-docker compose down --remove-orphans || true
-docker compose up -d --build postgres redis migrate api worker beat
-docker compose ps

docker-compose.yml

@@ -1,5 +1,5 @@
 x-app-env: &app-env
-  # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
+  # Всегда используем Postgres.
   IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
   IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
@@ -8,20 +8,28 @@ x-app-env: &app-env
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  # 0 => без лимита (в коде интерпретируется как None).
-  # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
+  # 0 = без лимита.
   CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
-  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
   CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
-  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
+  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
+  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
+  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
   IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
   IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
   IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
   IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
   IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
   IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
   IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
   IAAI_BROWSER_ENGINE: chromium
+  # Self-heal для worker.
+  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
+  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
+  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
+  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
+  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -42,7 +50,7 @@ x-worker-service: &worker-service
     - tokens_data:/data
 
 services:
-  # PostgreSQL
+  # PostgreSQL.
   postgres:
     image: postgres:16-alpine
     container_name: iaai-postgres
@@ -66,7 +74,7 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Redis (Celery broker)
+  # Redis.
   redis:
     image: redis:7-alpine
     container_name: iaai-redis
@@ -90,7 +98,7 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # DB migrations
+  # Миграции.
   migrate:
     <<: *app-service
     container_name: iaai-migrate
@@ -100,7 +108,7 @@ services:
         condition: service_healthy
     command: alembic upgrade head
 
-  # FastAPI
+  # API.
   api:
     <<: *app-service
     container_name: iaai-api
@@ -128,14 +136,11 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Celery Worker
+  # Worker.
   worker:
     <<: *worker-service
-    container_name: iaai-worker
     restart: unless-stopped
     init: true
-    mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
-    memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
     depends_on:
       migrate:
         condition: service_completed_successfully
@@ -144,11 +149,12 @@ services:
     stop_grace_period: 60s
     command: >
       celery -A iaai_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
+      --loglevel=info --concurrency=1 --pool=solo
       --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
     healthcheck:
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
+      # Проверка worker.
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
       interval: 60s
       timeout: 10s
       retries: 3
@@ -159,7 +165,7 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Celery Beat (периодический планировщик)
+  # Beat.
   beat:
     <<: *worker-service
     container_name: iaai-beat

entrypoint.sh

@@ -55,6 +55,35 @@ start_proxy_bridge_if_needed() {
   echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
 }
 
+start_self_heal_watchdog_if_worker() {
+  if ! is_worker_command "$@"; then
+    return 0
+  fi
+
+  # После reboot/restart контейнера файловая система контейнера сохраняется,
+  # поэтому stale pidfile может остаться от прошлого запуска и блокировать старт Celery.
+  # Удаляем его перед запуском worker-процесса.
+  rm -f /tmp/celery-worker.pid
+
+  if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then
+    echo "[entrypoint] Self-heal watchdog disabled"
+    return 0
+  fi
+
+  echo "[entrypoint] Starting self-heal watchdog for worker..."
+  python -m iaai_scraper.worker.self_heal &
+  SELF_HEAL_PID=$!
+  sleep 1
+
+  if ! kill -0 "${SELF_HEAL_PID}" 2>/dev/null; then
+    echo "[entrypoint] ERROR: self-heal watchdog failed to start"
+    exit 1
+  fi
+
+  echo "[entrypoint] Self-heal watchdog started (PID ${SELF_HEAL_PID})"
+}
+
 start_proxy_bridge_if_needed "$@"
+start_self_heal_watchdog_if_worker "$@"
 
 exec "$@"

iaai_scraper/browser/factory.py

@@ -15,7 +15,7 @@ logger = logging.getLogger("iaai_scraper.browser")
 
 
 def _build_init_script() -> str:
-    # Патч признаков автоматизации.
+    # Маскировка браузера.
     hardware_concurrency = random.choice([4, 8, 12, 16])
     device_memory = random.choice([4, 8, 16])
     languages = ["en-US", "en"]
@@ -69,11 +69,10 @@ class BrowserFactory:
         self.settings = settings
 
     def _resolve_engine(self) -> str:
-        # Выбор движка браузера.
+        # Выбор движка.
         engine = self.settings.browser_engine.strip().lower()
         if engine == "auto":
-            # Для IAAI в headless-режиме Chromium со stealth-скриптами
-            # значительно стабильнее Firefox по anti-bot.
+            # Для IAAI стабильнее Chromium.
             return "chromium"
         if engine in ("firefox", "chromium"):
             return engine
@@ -90,11 +89,11 @@ class BrowserFactory:
             if proxy_dict:
                 launch_kwargs["proxy"] = proxy_dict
                 logger.info("Using proxy: %s", self.settings.proxy.server)
-            # Параметры Firefox для headless-режима.
+            # Настройки Firefox.
             launch_kwargs["firefox_user_prefs"] = {
                 "dom.webdriver.enabled": False,
                 "useAutomationExtension": False,
-                # Базовые оптимизации для VPS.
+                # Базовые оптимизации.
                 "media.autoplay.default": 5,
                 "media.volume_scale": "0.0",
                 "media.audio.playback.standalone": False,
@@ -111,7 +110,7 @@ class BrowserFactory:
             logger.info("Launching Firefox (headless=%s)", self.settings.headless)
             return playwright.firefox.launch(**launch_kwargs)
 
-        # Путь запуска Chromium.
+        # Запуск Chromium.
         args = [
             "--disable-blink-features=AutomationControlled",
             "--no-default-browser-check",
@@ -154,7 +153,7 @@ class BrowserFactory:
         }
 
         if is_firefox:
-            # Заголовки и user-agent для Firefox.
+            # Заголовки Firefox.
             ctx_kwargs["user_agent"] = (
                 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
                 "Gecko/20100101 Firefox/128.0"
@@ -183,7 +182,7 @@ class BrowserFactory:
         context.set_default_navigation_timeout(self.settings.default_timeout_ms)
 
         if not is_firefox:
-            # Патчи маскировки для Chromium.
+            # Маскировка Chromium.
             context.add_init_script(_build_init_script())
             if stealth_sync:
                 context.on("page", lambda page: stealth_sync(page))
@@ -193,7 +192,7 @@ class BrowserFactory:
 
     @staticmethod
     def enable_resource_blocking(page) -> None:
-        # Блокируем тяжёлые ресурсы для ускорения загрузки страниц.
+        # Блокируем тяжёлые ресурсы.
         BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
         BLOCKED_URL_PATTERNS = (
             "google-analytics", "googletagmanager", "facebook.net",

iaai_scraper/browser/listing.py

@@ -3,12 +3,13 @@ import re
 import time
 from dataclasses import asdict, dataclass, field
 from typing import Any
-from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
+from urllib.parse import urljoin
 
 from playwright.sync_api import Page
 
 from .pace import HumanPacer
 from ..core.config import Settings
+from ..core.utils import first_non_empty
 
 logger = logging.getLogger("iaai_scraper.listing")
 VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
@@ -43,7 +44,6 @@ class ListingPageResult:
 
 
 class ListingCollector:
-	_DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE = 40
 	_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
 		"a[aria-label*='Next']",
 		"button[aria-label*='Next']",
@@ -103,33 +103,7 @@ class ListingCollector:
 			return None
 
 	@staticmethod
-	def _get_vehicle_link_fingerprint(page: Page, limit: int = 5) -> tuple[str, ...]:
-		try:
-			values = page.evaluate(
-				"""
-				(limit) => {
-					return Array.from(document.querySelectorAll("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"))
-						.map((el) => (el.getAttribute('href') || '').trim())
-						.filter(Boolean)
-						.slice(0, limit);
-				}
-				""",
-				limit,
-			)
-			if not isinstance(values, list):
-				return tuple()
-			return tuple(str(value) for value in values if value)
-		except Exception:
-			return tuple()
-
-	@staticmethod
-	def _wait_for_navigation_result(
-		page: Page,
-		old_first_href: str,
-		expected_page_number: int | None,
-		*,
-		old_fingerprint: tuple[str, ...] = (),
-	) -> bool:
+	def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
 		if old_first_href:
 			try:
 				page.wait_for_function(
@@ -143,12 +117,9 @@ class ListingCollector:
 			except Exception:
 				pass
 
-		new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
 		if expected_page_number is not None:
 			current_page = ListingCollector._get_current_page_number(page)
-			if current_page == expected_page_number and (
-				not old_fingerprint or (new_fingerprint and new_fingerprint != old_fingerprint)
-			):
+			if current_page == expected_page_number:
 				return True
 
 		try:
@@ -156,81 +127,33 @@ class ListingCollector:
 		except Exception:
 			pass
 
-		if not new_fingerprint:
-			new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
 		current_page = ListingCollector._get_current_page_number(page)
-		if new_fingerprint and old_fingerprint and new_fingerprint != old_fingerprint:
-			return current_page is None or expected_page_number is None or current_page == expected_page_number
 		if expected_page_number is not None and current_page == expected_page_number:
-			return not old_fingerprint
+			return True
 
-		return not old_first_href and (not old_fingerprint or bool(new_fingerprint))
-
-	def _extract_next_page_href(self, page: Page, expected_page_number: int | None = None) -> str | None:
-		try:
-			href = page.evaluate(
-				"""
-				(expectedPageNumber) => {
-					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
-					const disabled = (el) => {
-						if (!el) return true;
-						const cls = (el.getAttribute('class') || '').toLowerCase();
-						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
-						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
-					};
-
-					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
-						.filter((el) => visible(el) && !disabled(el));
-
-					const cleanHref = (el) => {
-						const href = (el?.getAttribute('href') || '').trim();
-						if (!href || href.startsWith('javascript:') || href.startsWith('#')) {
-							return '';
-						}
-						return href;
-					};
-
-					if (expectedPageNumber !== null && expectedPageNumber !== undefined) {
-						const numeric = controls.find((el) => {
-							const text = (el.textContent || '').trim();
-							return /^\d+$/.test(text) && parseInt(text, 10) === expectedPageNumber;
-						});
-						const numericHref = cleanHref(numeric);
-						if (numericHref) {
-							return numericHref;
-						}
-					}
-
-					const explicitNext = controls.find((el) => {
-						const text = (el.textContent || '').trim().toLowerCase();
-						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
-						const title = (el.getAttribute('title') || '').trim().toLowerCase();
-						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
-						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
-						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
-						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
-					});
-					return cleanHref(explicitNext);
-				}
-				""",
-				expected_page_number,
-			)
-			if not href:
-				return None
-			return urljoin(page.url or self.settings.home_url, str(href))
-		except Exception:
-			return None
+		return not old_first_href
 
 	@staticmethod
-	def _build_listing_page_url(url: str, page_number: int) -> str:
-		parsed = urlparse(url)
-		query_items = [
-			(key, value)
-			for key, value in parse_qsl(parsed.query, keep_blank_values=True)
-			if key.lower() not in {"page", "pagenumber", "currentpage"}
-		]
-		query_items.append(("page", str(page_number)))
-		return urlunparse(parsed._replace(query=urlencode(query_items)))
+	def has_page_number(page: Page, target_page_number: int) -> bool:
+		try:
+			return bool(page.evaluate(
+				"""
+				(targetPageNumber) => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+						return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
+					});
+				}
+				""",
+				target_page_number,
+			))
+		except Exception:
+			return False
 
 	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
 		url = url_override or self.settings.listing.cars_url
@@ -301,31 +224,6 @@ class ListingCollector:
 			# Короткая пауза вместо длинного sleep.
 			time.sleep(1.0)
 
-	def _get_listing_html(self, page: Page, page_number: int) -> str:
-		try:
-			return page.locator("html").inner_html(timeout=5_000)
-		except Exception as exc:
-			logger.warning("listing html read failed on page %d: %s", page_number, exc)
-			return ""
-
-	def _has_next_page_from_html(self, html: str, current_page_number: int | None = None) -> bool:
-		if not html:
-			return False
-		normalized = html.replace("\\/", "/")
-		if re.search(
-			r"rel\s*=\s*['\"]next['\"]|aria-label\s*=\s*['\"][^'\"]*next|title\s*=\s*['\"][^'\"]*next|class\s*=\s*['\"][^'\"]*next|icon-arrow-right|arrow-right|>\s*next\s*<|>\s*[›»>]\s*<",
-			normalized,
-			re.IGNORECASE,
-		):
-			return True
-		if current_page_number is not None:
-			next_page = current_page_number + 1
-			if re.search(rf">\s*{next_page}\s*<", normalized, re.IGNORECASE):
-				return True
-			if re.search(rf"page={next_page}(?:\D|$)", normalized, re.IGNORECASE):
-				return True
-		return False
-
 	def apply_filters(
 		self,
 		page: Page,
@@ -403,30 +301,74 @@ class ListingCollector:
 		return False
 
 	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
-		# Для IAAI HTML/hydration-извлечение стабильнее, чем прямой DOM eval.
+		# Считываем ссылки одним проходом по DOM.
 		self._accept_cookie_banner(page)
 		self._wait_for_listing_content(page)
+		try:
+			raw_items = page.eval_on_selector_all(
+				"a[href], [data-href], [href]",
+				"""
+				(nodes) => nodes.map((node) => ({
+					href:
+						node.getAttribute('href') ||
+						node.getAttribute('data-href') ||
+						node.getAttribute('data-url') ||
+						'',
+					title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
+					text: (node.textContent || '').trim(),
+				}))
+				""",
+			)
+		except Exception as exc:
+			logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
+			raw_items = []
+		total = min(len(raw_items), self.settings.listing.page_link_limit)
 		links: list[ListingVehicleLink] = []
 		seen: set[str] = set()
-		html = self._get_listing_html(page, page_number)
-		for absolute, lot_number in self._extract_vehicle_links_from_html(html):
+		for idx in range(total):
+			item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
+			href = str(item.get("href") or "")
+			match = VEHICLE_HREF_RE.search(href)
+			if not match:
+				continue
+			lot_number = match.group(1)
+			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
 			seen.add(absolute)
-			links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
+			title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
+			links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
 			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
 
-		if links:
-			logger.info(
-				"Page %d: recovered %d vehicle links from HTML fallback",
-				page_number,
-				len(links),
-			)
+		# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
+		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
+		if not links:
+			try:
+				page.wait_for_timeout(1_500)
+			except Exception:
+				pass
+			try:
+				html = page.content()
+			except Exception as exc:
+				logger.debug("page.content() failed on page %d: %s", page_number, exc)
+				html = ""
 
-		next_page_detected = self._has_next_page_from_html(html, current_page_number=page_number)
-		if not next_page_detected and not html:
-			next_page_detected = self._has_next_page(page)
+			for absolute, lot_number in self._extract_vehicle_links_from_html(html):
+				if absolute in seen:
+					continue
+				seen.add(absolute)
+				links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
+				if len(links) >= self.settings.listing.max_vehicles_per_run:
+					break
+
+			if links:
+				logger.info(
+					"Page %d: recovered %d vehicle links from HTML fallback",
+					page_number,
+					len(links),
+				)
+		next_page_detected = self._has_next_page(page)
 		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
 
 	def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
@@ -453,59 +395,12 @@ class ListingCollector:
 	def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
 		# Запоминаем первую ссылку текущей страницы для определения смены контента.
 		old_first_href = ""
-		old_fingerprint: tuple[str, ...] = tuple()
 		try:
 			first_link = page.locator(VEHICLE_LINK_SELECTOR).first
 			if first_link.count() > 0:
 				old_first_href = first_link.get_attribute("href") or ""
 		except Exception:
 			pass
-		old_fingerprint = self._get_vehicle_link_fingerprint(page)
-
-		if expected_page_number is not None and expected_page_number > 1:
-			direct_page_url = self._build_listing_page_url(
-				page.url or self.settings.listing.cars_url,
-				expected_page_number,
-			)
-			try:
-				logger.debug("Navigating directly to listing page %d via URL: %s", expected_page_number, direct_page_url)
-				page.goto(direct_page_url, wait_until="domcontentloaded", timeout=15_000)
-				if self._wait_for_navigation_result(
-					page,
-					old_first_href,
-					expected_page_number,
-					old_fingerprint=old_fingerprint,
-				):
-					self.pacer.after_page_change()
-					return True
-			except Exception as exc:
-				logger.debug("Direct page-number navigation failed for page %d via %s: %s", expected_page_number, direct_page_url, exc)
-
-		next_href = self._extract_next_page_href(page, expected_page_number)
-		if next_href:
-			try:
-				logger.debug("Navigating directly to next listing page: %s", next_href)
-				page.goto(next_href, wait_until="domcontentloaded", timeout=15_000)
-				if self._wait_for_navigation_result(
-					page,
-					old_first_href,
-					expected_page_number,
-					old_fingerprint=old_fingerprint,
-				):
-					self.pacer.after_page_change()
-					return True
-			except Exception as exc:
-				logger.debug("Direct next-page navigation failed for %s: %s", next_href, exc)
-
-		if (
-			expected_page_number is not None
-			and expected_page_number >= self._DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE
-		):
-			logger.warning(
-				"Deep pagination direct navigation failed for page %d; skipping flaky UI pagination fallbacks",
-				expected_page_number,
-			)
-			return False
 
 		for selector in self._NEXT_PAGE_SELECTORS:
 			locator = page.locator(selector).first
@@ -525,12 +420,7 @@ class ListingCollector:
 			except Exception:
 				continue
 
-			if self._wait_for_navigation_result(
-				page,
-				old_first_href,
-				expected_page_number,
-				old_fingerprint=old_fingerprint,
-			):
+			if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
 				self.pacer.after_page_change()
 				return True
 
@@ -590,12 +480,7 @@ class ListingCollector:
 				"""
 			))
 			if clicked:
-				if self._wait_for_navigation_result(
-					page,
-					old_first_href,
-					expected_page_number,
-					old_fingerprint=old_fingerprint,
-				):
+				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
 					self.pacer.after_page_change()
 					return True
 		except Exception as exc:
@@ -626,12 +511,7 @@ class ListingCollector:
 				"""
 			))
 			if clicked:
-				if self._wait_for_navigation_result(
-					page,
-					old_first_href,
-					expected_page_number,
-					old_fingerprint=old_fingerprint,
-				):
+				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
 					self.pacer.after_page_change()
 					return True
 		except Exception as exc:
@@ -716,18 +596,10 @@ class ListingCollector:
 				len(all_links) >= self.settings.listing.max_vehicles_per_run
 				or self.settings.listing.collect_current_page_only
 				or not self.settings.listing.include_pagination
+				or not page_result.next_page_detected
 			):
 				break
-
-			blind_page_probe = not page_result.next_page_detected
-			if not self.go_to_next_page(page, expected_page_number=page_number + 1):
-				if blind_page_probe:
-					logger.info(
-						"Stopping pagination on page %d: direct page probe for %d failed and no next-page control was detected",
-						page_number,
-						page_number + 1,
-					)
-					break
+			if not self.go_to_next_page(page):
 				break
 
 		return {
@@ -775,8 +647,22 @@ class ListingCollector:
 	@staticmethod
 	def _has_next_page(page: Page) -> bool:
 		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
-			if page.locator(selector).count() > 0:
-				return True
+			locator = page.locator(selector)
+			count = locator.count()
+			if count == 0:
+				continue
+			# Проверяем, что хотя бы один элемент не disabled.
+			# Disabled "Next" на последней странице не означает наличия следующей.
+			for i in range(min(count, 3)):
+				try:
+					el = locator.nth(i)
+					disabled_attr = el.get_attribute("disabled", timeout=300)
+					aria_disabled = el.get_attribute("aria-disabled", timeout=300)
+					cls = (el.get_attribute("class", timeout=300) or "").lower()
+					if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
+						return True
+				except Exception:
+					continue
 		try:
 			return bool(page.evaluate(
 				"""

iaai_scraper/cli.py

@@ -38,17 +38,6 @@ def build_parser() -> argparse.ArgumentParser:
     sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
     sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
 
-    # Новые команды для ingestion pipeline
-    subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap")
-
-    fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates")
-    fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process")
-
-    enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots")
-    enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process")
-
-    subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)")
-
     return parser
 
 
@@ -75,39 +64,6 @@ def main() -> None:
             data = scraper.scrape_vehicle_detail(args.vehicle_url)
         elif args.command == "sync-vehicle":
             data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
-        elif args.command == "sync-listing":
-            only_new = None if args.only_new is None else args.only_new == "true"
-            data = scraper.sync_listing(
-                make=args.make,
-                model=args.model,
-                lane=args.lane,
-                limit=args.limit,
-                only_new=only_new,
-            )
-        elif args.command == "discover-vehicles":
-            from .discovery_service import DiscoveryService
-            discovery = DiscoveryService()
-            count = discovery.discover_new_vehicles()
-            print(f"Discovered {count} new vehicle candidates")
-            return
-        elif args.command == "fetch-pending":
-            from .fetch_service import FetchService
-            fetch = FetchService()
-            count = fetch.process_pending_candidates(limit=args.limit)
-            print(f"Successfully fetched {count} candidates")
-            return
-        elif args.command == "enrich-snapshots":
-            from .enrichment_service import EnrichmentService
-            enrichment = EnrichmentService()
-            count = enrichment.process_unparsed_snapshots(limit=args.limit)
-            print(f"Successfully enriched {count} snapshots")
-            return
-        elif args.command == "run-pipeline":
-            from .scheduler_service import SchedulerService
-            scheduler = SchedulerService()
-            scheduler.run_full_pipeline()
-            print("Pipeline completed")
-            return
         else:
             only_new = None if args.only_new is None else args.only_new == "true"
             data = scraper.sync_listing(

iaai_scraper/core/config.py

@@ -106,7 +106,6 @@ class ListingConfig:
     cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
     max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
     max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
-    resume_max_nav_pages: int = _env_int("IAAI_LISTING_RESUME_MAX_NAV_PAGES", 90)
     page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
     include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
     collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
@@ -119,22 +118,6 @@ class ListingConfig:
     listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
 
 
-@dataclass(slots=True)
-class DiscoveryConfig:
-    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
-    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
-    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 3000)
-    sitemap_timeout_seconds: int = _env_int("IAAI_SITEMAP_TIMEOUT_SECONDS", 30)
-    sitemap_retry_attempts: int = _env_int("IAAI_SITEMAP_RETRY_ATTEMPTS", 4)
-    sitemap_retry_backoff_seconds: float = _env_float("IAAI_SITEMAP_RETRY_BACKOFF_SECONDS", 1.25)
-    sitemap_direct_probe_limit: int = _env_int("IAAI_SITEMAP_DIRECT_PROBE_LIMIT", 12)
-    sitemap_direct_probe_stop_after_misses: int = _env_int(
-        "IAAI_SITEMAP_DIRECT_PROBE_STOP_AFTER_MISSES",
-        3,
-    )
-    sitemap_use_curl_cffi: bool = _env_bool("IAAI_SITEMAP_USE_CURL_CFFI", True)
-
-
 # Список брендов IAAI для автоматической сегментации.
 # Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
 IAAI_DEFAULT_MAKES: tuple[str, ...] = (
@@ -226,6 +209,16 @@ class RedisConfig:
     health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
 
 
+# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
+    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
+    always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", True)
+
+
 # --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
 
 @dataclass(slots=True)
@@ -242,7 +235,7 @@ class CeleryConfig:
     beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
     batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
     parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
-    parallel_segments: bool = _env_bool("IAAI_PARALLEL_SEGMENTS", False)
+    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
     block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
 
 
@@ -297,11 +290,11 @@ class Settings:
     capture: CaptureConfig = field(default_factory=CaptureConfig)
     pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
     listing: ListingConfig = field(default_factory=ListingConfig)
-    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
     database: DatabaseConfig = field(default_factory=DatabaseConfig)
     redis: RedisConfig = field(default_factory=RedisConfig)
     celery: CeleryConfig = field(default_factory=CeleryConfig)
     proxy: ProxyConfig = field(default_factory=ProxyConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
 
     @property
     def parallel_tabs(self) -> int:

iaai_scraper/discovery/__init__.py

@@ -1,17 +1,15 @@
 from .sitemap import (
-    DEFAULT_SITEMAP_INDEX_URL,
     SitemapDiscoveryError,
-    SitemapBlockedError,
     SitemapDiscoveryResult,
+    SitemapDiscoveryStats,
     discover_vehicle_urls_from_sitemap,
     discover_vehicle_urls_from_sitemap_with_stats,
 )
 
 __all__ = [
-    "DEFAULT_SITEMAP_INDEX_URL",
-    "SitemapBlockedError",
     "SitemapDiscoveryError",
     "SitemapDiscoveryResult",
+    "SitemapDiscoveryStats",
     "discover_vehicle_urls_from_sitemap",
     "discover_vehicle_urls_from_sitemap_with_stats",
 ]

iaai_scraper/discovery/sitemap.py

@@ -3,222 +3,27 @@ from __future__ import annotations
 import gzip
 import io
 import logging
-import random
 import re
-import time
-import xml.etree.ElementTree as ET
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Iterable
-from urllib.error import HTTPError, URLError
-from urllib.parse import urljoin, urlsplit, urlunsplit
-from urllib.request import Request, urlopen
-
-from ..core.config import Settings
-
-try:
-    from curl_cffi import requests as curl_requests
-except ImportError:  # pragma: no cover - optional runtime dependency guard
-    curl_requests = None
+from urllib.parse import urlsplit, urlunsplit
+from urllib.request import Request, urlopen, ProxyHandler, build_opener
+import xml.etree.ElementTree as ET
 
 logger = logging.getLogger("iaai_scraper.discovery.sitemap")
 
 DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+_SITEMAP_TIMEOUT_SECONDS = 30
 _LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
-_XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL)
-_BLOCK_MARKERS = (
-    b"pardon our interruption",
-    b"incapsula",
-    b"please stand by",
-    b"_incapsula_resource",
-    b"as you were browsing something about your browser",
-)
-_HTML_MARKERS = (b"<html", b"<!doctype html", b"<script", b"document.getelementsbyclassname")
-_CURL_IMPERSONATE_CHOICES = ("chrome136", "chrome133", "chrome131", "safari184")
 
 
 class SitemapDiscoveryError(RuntimeError):
     pass
 
 
-class SitemapBlockedError(SitemapDiscoveryError):
-    pass
-
-
-@dataclass(slots=True)
-class SitemapFetchResult:
-    url: str
-    payload: bytes
-    source: str
-    blocked: bool = False
-    status_code: int | None = None
-    content_type: str | None = None
-
-
-@dataclass(slots=True)
-class SitemapDiscoveryStats:
-    transport: str
-    fetched_sitemaps: int = 0
-    blocked_sitemaps: int = 0
-    malformed_sitemaps: int = 0
-    direct_probe_hits: int = 0
-    direct_probe_misses: int = 0
-
-
-@dataclass(slots=True)
-class SitemapDiscoveryResult:
-    vehicle_urls: list[str]
-    stats: SitemapDiscoveryStats
-
-
-_DEFAULT_HEADERS = {
-    "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
-    "Accept-Encoding": "gzip, deflate, br",
-    "Accept-Language": "en-US,en;q=0.9",
-    "Cache-Control": "no-cache",
-    "Pragma": "no-cache",
-    "Upgrade-Insecure-Requests": "1",
-}
-
-
-class _SitemapDownloader:
-    def __init__(self, settings: Settings) -> None:
-        self.settings = settings
-        self.discovery = settings.discovery
-        self.proxy_url = settings.proxy.server
-        self.proxy_auth = None
-        if settings.proxy.username:
-            password = settings.proxy.password or ""
-            self.proxy_auth = (settings.proxy.username, password)
-        self._transport_name = self._resolve_transport_name()
-
-    @property
-    def transport_name(self) -> str:
-        return self._transport_name
-
-    def _resolve_transport_name(self) -> str:
-        if self.discovery.sitemap_use_curl_cffi and curl_requests is not None:
-            return "curl_cffi"
-        return "urllib"
-
-    def fetch(self, url: str) -> SitemapFetchResult:
-        attempts = max(1, int(self.discovery.sitemap_retry_attempts))
-        last_exc: Exception | None = None
-        for attempt in range(1, attempts + 1):
-            try:
-                result = self._fetch_once(url)
-                if result.blocked:
-                    raise SitemapBlockedError(f"Anti-bot page returned for {url}")
-                return result
-            except SitemapBlockedError as exc:
-                last_exc = exc
-                logger.warning(
-                    "Sitemap fetch blocked (attempt %d/%d, transport=%s): %s",
-                    attempt,
-                    attempts,
-                    self.transport_name,
-                    url,
-                )
-            except Exception as exc:
-                last_exc = exc
-                logger.warning(
-                    "Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s",
-                    attempt,
-                    attempts,
-                    self.transport_name,
-                    url,
-                    exc,
-                )
-            if attempt >= attempts:
-                break
-            time.sleep(self._backoff_delay(attempt))
-        if last_exc is None:
-            raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}")
-        if isinstance(last_exc, SitemapDiscoveryError):
-            raise last_exc
-        raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc
-
-    def _fetch_once(self, url: str) -> SitemapFetchResult:
-        if self.transport_name == "curl_cffi":
-            return self._fetch_with_curl_cffi(url)
-        return self._fetch_with_urllib(url)
-
-    def _build_headers(self) -> dict[str, str]:
-        headers = dict(_DEFAULT_HEADERS)
-        headers["User-Agent"] = self.settings.fingerprint.user_agent
-        return headers
-
-    def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult:
-        assert curl_requests is not None
-        response = curl_requests.get(
-            url,
-            headers=self._build_headers(),
-            timeout=self.discovery.sitemap_timeout_seconds,
-            impersonate=random.choice(_CURL_IMPERSONATE_CHOICES),
-            proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None,
-            proxy_auth=self.proxy_auth,
-            allow_redirects=True,
-        )
-        payload = self._decode_payload(
-            payload=response.content,
-            encoding=(response.headers.get("Content-Encoding") or ""),
-            url=url,
-        )
-        blocked = _looks_like_block_page(payload)
-        return SitemapFetchResult(
-            url=url,
-            payload=payload,
-            source="curl_cffi",
-            blocked=blocked,
-            status_code=int(response.status_code),
-            content_type=response.headers.get("Content-Type"),
-        )
-
-    def _fetch_with_urllib(self, url: str) -> SitemapFetchResult:
-        request = Request(url, headers=self._build_headers())
-        try:
-            with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response:
-                payload = response.read()
-                decoded = self._decode_payload(
-                    payload=payload,
-                    encoding=(response.headers.get("Content-Encoding") or ""),
-                    url=url,
-                )
-                return SitemapFetchResult(
-                    url=url,
-                    payload=decoded,
-                    source="urllib",
-                    blocked=_looks_like_block_page(decoded),
-                    status_code=getattr(response, "status", None),
-                    content_type=response.headers.get("Content-Type"),
-                )
-        except HTTPError as exc:
-            payload = exc.read() if hasattr(exc, "read") else b""
-            decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url)
-            blocked = exc.code in {403, 429} or _looks_like_block_page(decoded)
-            if blocked:
-                raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc
-            raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc
-        except URLError as exc:
-            raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc
-
-    @staticmethod
-    def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes:
-        normalized = (encoding or "").lower().strip()
-        if normalized == "gzip" or url.lower().endswith(".gz"):
-            try:
-                return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
-            except OSError:
-                return payload
-        return payload
-
-    def _backoff_delay(self, attempt: int) -> float:
-        base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds))
-        jitter = random.uniform(0.05, 0.35)
-        return base * (2 ** (attempt - 1)) + jitter
-
-
 def _is_vehicle_sitemap_url(url: str) -> bool:
     lowered = url.strip().lower()
+    # Берём только sitemap с авто.
     if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
         return False
     return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
@@ -229,21 +34,38 @@ def _normalize_vehicle_url(url: str) -> str:
     return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
 
 
+def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
+    request = Request(
+        url,
+        headers={
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+            ),
+            "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
+            "Accept-Encoding": "gzip",
+        },
+    )
+    if proxy_url:
+        handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
+        opener = build_opener(handler)
+        response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    else:
+        response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    with response:
+        payload = response.read()
+        encoding = str(response.headers.get("Content-Encoding") or "").lower()
+        if encoding == "gzip" or url.lower().endswith(".gz"):
+            return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
+        return payload
+
+
 def _local_name(tag: str) -> str:
     if "}" in tag:
         return tag.rsplit("}", 1)[1]
     return tag
 
 
-def _looks_like_block_page(payload: bytes) -> bool:
-    sample = payload[:8192].lower()
-    if any(marker in sample for marker in _BLOCK_MARKERS):
-        return True
-    if any(marker in sample for marker in _HTML_MARKERS) and b"<loc>" not in sample:
-        return True
-    return False
-
-
 def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
     for match in _LOC_TAG_RE.finditer(xml_bytes):
         try:
@@ -255,13 +77,6 @@ def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
 
 
 def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
-    if _looks_like_block_page(xml_bytes):
-        raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML")
-    if not _XML_PREFIX_RE.search(xml_bytes[:256]):
-        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
-        if fallback_values:
-            yield from fallback_values
-            return
     try:
         root = ET.fromstring(xml_bytes)
     except ET.ParseError as exc:
@@ -303,139 +118,93 @@ def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
     return any("/vehicledetail/" in url.lower() for url in urls)
 
 
-def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]:
-    base_dir = index_url.rsplit("/", 1)[0] + "/"
-    return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)]
-
-
-def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]:
+def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
     logger.info("Downloading sitemap index: %s", index_url)
-    try:
-        index_result = downloader.fetch(index_url)
-        sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)]
-        if sitemap_urls:
-            return sitemap_urls
-    except SitemapBlockedError as exc:
-        stats.blocked_sitemaps += 1
-        logger.warning("Sitemap index blocked, switching to direct probe: %s", exc)
-    except SitemapDiscoveryError as exc:
-        stats.malformed_sitemaps += 1
-        logger.warning("Sitemap index unusable, switching to direct probe: %s", exc)
-
-    logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe")
-    return _probe_direct_sitemap_urls(index_url, downloader, stats)
-
-
-def _probe_direct_sitemap_urls(
-    index_url: str,
-    downloader: _SitemapDownloader,
-    stats: SitemapDiscoveryStats,
-) -> list[str]:
-    discovered: list[str] = []
-    consecutive_misses = 0
-    probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit)
-
-    for sitemap_url in probe_urls:
-        try:
-            result = downloader.fetch(sitemap_url)
-            raw_urls = list(_iter_loc_values(result.payload))
-        except SitemapBlockedError:
-            stats.blocked_sitemaps += 1
-            consecutive_misses += 1
-            stats.direct_probe_misses += 1
-            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
-                break
-            continue
-        except SitemapDiscoveryError:
-            consecutive_misses += 1
-            stats.direct_probe_misses += 1
-            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
-                break
-            continue
-
-        if not raw_urls:
-            consecutive_misses += 1
-            stats.direct_probe_misses += 1
-            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
-                break
-            continue
-
-        consecutive_misses = 0
-        stats.direct_probe_hits += 1
-        discovered.append(sitemap_url)
-
-    return discovered
-
-
-def _collect_vehicle_urls_from_sitemap(
-    sitemap_url: str,
-    downloader: _SitemapDownloader,
-    stats: SitemapDiscoveryStats,
-) -> list[str]:
-    logger.info("Downloading sitemap: %s", sitemap_url)
-    try:
-        result = downloader.fetch(sitemap_url)
-        raw_urls = list(_iter_loc_values(result.payload))
-    except SitemapBlockedError as exc:
-        stats.blocked_sitemaps += 1
-        logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc)
-        return []
-    except SitemapDiscoveryError as exc:
-        stats.malformed_sitemaps += 1
-        logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
-        return []
-
-    stats.fetched_sitemaps += 1
-    vehicle_urls = _filter_vehicle_urls(raw_urls)
-    if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
-        logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
-        return []
-
-    logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
-    return vehicle_urls
-
-
-def discover_vehicle_urls_from_sitemap(
-    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
-    *,
-    settings: Settings | None = None,
-) -> list[str]:
-    result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings)
-    return result.vehicle_urls
-
-
-
-def discover_vehicle_urls_from_sitemap_with_stats(
-    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
-    *,
-    settings: Settings | None = None,
-) -> SitemapDiscoveryResult:
-    runtime_settings = settings or Settings()
-    downloader = _SitemapDownloader(runtime_settings)
-    stats = SitemapDiscoveryStats(transport=downloader.transport_name)
-
-    sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
     if not sitemap_urls:
-        raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs")
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
 
     all_vehicle_urls: list[str] = []
     for sitemap_url in sitemap_urls:
-        all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats))
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
 
     deduped = _filter_vehicle_urls(all_vehicle_urls)
     if not deduped:
-        raise SitemapDiscoveryError(
-            "No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue"
-        )
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return deduped
 
-    logger.info(
-        "Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)",
-        len(deduped),
-        stats.transport,
-        stats.fetched_sitemaps,
-        stats.blocked_sitemaps,
-        stats.malformed_sitemaps,
-        stats.direct_probe_hits,
-        stats.direct_probe_misses,
-    )
+
+@dataclass
+class SitemapDiscoveryStats:
+    transport: str = "http"
+    fetched_sitemaps: int = 0
+    blocked_sitemaps: int = 0
+    malformed_sitemaps: int = 0
+    direct_probe_hits: int = 0
+    direct_probe_misses: int = 0
+
+
+@dataclass
+class SitemapDiscoveryResult:
+    vehicle_urls: list[str] = field(default_factory=list)
+    stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
+
+
+def discover_vehicle_urls_from_sitemap_with_stats(
+    settings=None,
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+) -> SitemapDiscoveryResult:
+    """Возвращает URL и статистику."""
+    proxy_url = None
+    if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
+        proxy_url = settings.proxy.server
+    stats = SitemapDiscoveryStats(transport="http")
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+            stats.fetched_sitemaps += 1
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            stats.malformed_sitemaps += 1
+            continue
+        except Exception as exc:
+            logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
+            stats.blocked_sitemaps += 1
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
     return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

iaai_scraper/discovery_service.py

@@ -1,129 +0,0 @@
-import logging
-from datetime import datetime, timezone
-from typing import List
-
-from sqlalchemy.orm import Session
-
-from .core.config import settings
-from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError
-from .storage.db import get_db_session
-from .storage.models import VehicleCandidate
-
-logger = logging.getLogger("iaai_scraper.discovery_service")
-
-
-class DiscoveryService:
-    """Сервис для обнаружения новых автомобилей через sitemap IAAI."""
-
-    def __init__(self):
-        self.db: Session = get_db_session()
-
-    def discover_new_vehicles(self, max_urls: int = None) -> int:
-        """
-        Обнаруживает новые URL автомобилей и добавляет их в vehicle_candidates.
-
-        Args:
-            max_urls: Максимальное количество URL для обработки (None = все)
-
-        Returns:
-            Количество добавленных кандидатов
-        """
-        try:
-            logger.info("Starting vehicle discovery from sitemap")
-            result = discover_vehicle_urls_from_sitemap_with_stats()
-
-            if not result.urls:
-                logger.info("No new URLs discovered")
-                return 0
-
-            # Ограничиваем количество URL
-            urls_to_process = result.urls[:max_urls] if max_urls else result.urls
-            logger.info(f"Discovered {len(urls_to_process)} potential vehicle URLs (from {len(result.urls)} total)")
-
-            # Фильтруем уже существующие кандидаты
-            existing_urls = self._get_existing_candidate_urls(urls_to_process)
-            new_urls = [url for url in urls_to_process if url not in existing_urls]
-
-            if not new_urls:
-                logger.info("All discovered URLs already exist as candidates")
-                return 0
-
-            # Добавляем новых кандидатов
-            added_count = self._add_candidates(new_urls)
-            logger.info(f"Added {added_count} new vehicle candidates")
-
-            return added_count
-
-        except SitemapDiscoveryError as e:
-            logger.error(f"Sitemap discovery failed: {e}")
-            raise
-        except Exception as e:
-            logger.error(f"Unexpected error during discovery: {e}")
-            raise
-
-    def _get_existing_candidate_urls(self, urls: List[str]) -> set:
-        """Получает множество уже существующих URL кандидатов."""
-        if not urls:
-            return set()
-
-        # Разбиваем на батчи для эффективности
-        batch_size = 1000
-        existing = set()
-
-        for i in range(0, len(urls), batch_size):
-            batch = urls[i:i + batch_size]
-            result = self.db.query(VehicleCandidate.url).filter(
-                VehicleCandidate.url.in_(batch)
-            ).all()
-            existing.update(row[0] for row in result)
-
-        return existing
-
-    def _add_candidates(self, urls: List[str]) -> int:
-        """Добавляет новые кандидаты в базу данных."""
-        now = datetime.now(timezone.utc)
-        candidates = [
-            VehicleCandidate(
-                url=url,
-                discovered_at=now,
-                status="pending",
-                priority=0
-            )
-            for url in urls
-        ]
-
-        self.db.add_all(candidates)
-        self.db.commit()
-
-        return len(candidates)
-
-    def get_pending_candidates(self, limit: int = 100) -> List[VehicleCandidate]:
-        """Получает кандидатов в статусе 'pending' для обработки."""
-        return self.db.query(VehicleCandidate).filter(
-            VehicleCandidate.status == "pending"
-        ).order_by(VehicleCandidate.priority.desc(), VehicleCandidate.discovered_at).limit(limit).all()
-
-    def mark_candidate_processing(self, candidate_id: int):
-        """Помечает кандидата как обрабатываемого."""
-        self.db.query(VehicleCandidate).filter(
-            VehicleCandidate.id == candidate_id
-        ).update({
-            "status": "processing",
-            "last_attempt_at": datetime.now(timezone.utc),
-            "attempts": VehicleCandidate.attempts + 1
-        })
-        self.db.commit()
-
-    def mark_candidate_processed(self, candidate_id: int):
-        """Помечает кандидата как обработанного."""
-        self.db.query(VehicleCandidate).filter(
-            VehicleCandidate.id == candidate_id
-        ).update({"status": "processed"})
-        self.db.commit()
-
-    def mark_candidate_failed(self, candidate_id: int):
-        """Помечает кандидата как неудачного."""
-        self.db.query(VehicleCandidate).filter(
-            VehicleCandidate.id == candidate_id
-        ).update({"status": "failed"})
-        self.db.commit()

iaai_scraper/enrichment_service.py

@@ -1,140 +0,0 @@
-import json
-import logging
-from datetime import datetime, timezone
-from typing import Optional
-
-from sqlalchemy.orm import Session
-
-from .core.config import settings
-from .parsing.mapper import CarMapper
-from .parsing.parser import VehicleParser
-from .storage.db import get_db_session
-from .storage.models import VehicleRawSnapshot, VehicleParseResult, Car
-from .storage.schemas import CarRecord
-
-logger = logging.getLogger("iaai_scraper.enrichment_service")
-
-
-class EnrichmentService:
-    """Сервис для парсинга сырых данных и обогащения автомобилей."""
-
-    def __init__(self):
-        self.db: Session = get_db_session()
-        self.parser = VehicleParser()
-        self.mapper = CarMapper()
-
-    def enrich_vehicle(self, snapshot: VehicleRawSnapshot) -> Optional[VehicleParseResult]:
-        """
-        Парсит snapshot и сохраняет результат.
-
-        Args:
-            snapshot: Raw snapshot для парсинга
-
-        Returns:
-            VehicleParseResult если парсинг успешен
-        """
-        logger.info(f"Enriching snapshot {snapshot.id} for candidate {snapshot.candidate_id}")
-
-        if not snapshot.success or not snapshot.raw_data:
-            logger.warning(f"Snapshot {snapshot.id} has no data to parse")
-            return self._save_parse_result(snapshot.id, False, None, "No raw data available")
-
-        try:
-            # Парсим данные
-            parsed_data = self._parse_raw_data(snapshot.raw_data)
-            if not parsed_data:
-                return self._save_parse_result(snapshot.id, False, None, "Parsing failed")
-
-            # Маппим в CarRecord
-            car_record = self._map_to_car_record(parsed_data)
-            if not car_record:
-                return self._save_parse_result(snapshot.id, False, None, "Mapping failed")
-
-            # Сохраняем в cars таблицу
-            self._save_car(car_record)
-
-            # Сохраняем успешный результат парсинга
-            return self._save_parse_result(snapshot.id, True, json.dumps(parsed_data))
-
-        except Exception as e:
-            error_msg = f"Unexpected error during enrichment: {str(e)}"
-            logger.error(f"Error enriching snapshot {snapshot.id}: {error_msg}")
-            return self._save_parse_result(snapshot.id, False, None, error_msg)
-
-    def _parse_raw_data(self, raw_data: str) -> Optional[dict]:
-        """Парсит сырые данные в словарь."""
-        try:
-            # Если это JSON от browser capture
-            if raw_data.strip().startswith('{'):
-                data = json.loads(raw_data)
-                # Извлекаем vehicle_summary из scrape result
-                return data.get("vehicle_summary", {})
-
-            # Если HTML, используем VehicleParser
-            parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
-            return parsed.get("vehicle_summary", {})
-
-        except json.JSONDecodeError:
-            # Если не JSON, пробуем как HTML
-            try:
-                parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
-                return parsed.get("vehicle_summary", {})
-            except Exception:
-                return None
-
-    def _map_to_car_record(self, parsed_data: dict) -> Optional[CarRecord]:
-        """Маппит parsed data в CarRecord."""
-        try:
-            # Используем CarMapper
-            payload_insights = {}  # TODO: extract from raw data if needed
-            return self.mapper.map_to_car_record("dummy_url", parsed_data, payload_insights)
-        except Exception as e:
-            logger.error(f"Mapping failed: {e}")
-            return None
-
-    def _save_car(self, car_record: CarRecord):
-        """Сохраняет CarRecord в базу данных."""
-        # Используем PersistenceService
-        from .storage.db import PersistenceService
-        from .core.config import settings
-        persistence = PersistenceService(settings)
-        persistence.create_tables()
-        upsert_result = persistence.upsert_car(car_record)
-        logger.info(f"Car upserted: {upsert_result}")
-
-    def _save_parse_result(self, snapshot_id: int, success: bool,
-                          parsed_data: Optional[str], error_message: Optional[str] = None) -> VehicleParseResult:
-        """Сохраняет результат парсинга."""
-        result = VehicleParseResult(
-            snapshot_id=snapshot_id,
-            success=success,
-            parsed_data=parsed_data,
-            error_message=error_message
-        )
-        self.db.add(result)
-        self.db.commit()
-        self.db.refresh(result)
-        return result
-
-    def process_unparsed_snapshots(self, limit: int = 10) -> int:
-        """
-        Обрабатывает snapshots без результатов парсинга.
-
-        Returns:
-            Количество успешно обогащенных snapshots
-        """
-        # Находим snapshots без parse results
-        snapshots = self.db.query(VehicleRawSnapshot).outerjoin(
-            VehicleParseResult, VehicleRawSnapshot.id == VehicleParseResult.snapshot_id
-        ).filter(
-            VehicleRawSnapshot.success == True,
-            VehicleParseResult.id.is_(None)
-        ).limit(limit).all()
-
-        enriched_count = 0
-        for snapshot in snapshots:
-            result = self.enrich_vehicle(snapshot)
-            if result and result.success:
-                enriched_count += 1
-
-        return enriched_count

iaai_scraper/fetch_service.py

@@ -1,119 +0,0 @@
-import logging
-from datetime import datetime, timezone
-from typing import Optional
-
-from sqlalchemy.orm import Session
-
-from .core.config import settings
-from .scraper import IAAIScraper
-from .storage.db import get_db_session
-from .storage.models import VehicleCandidate, VehicleRawSnapshot
-
-logger = logging.getLogger("iaai_scraper.fetch_service")
-
-
-class FetchService:
-    """Сервис для захвата сырых данных автомобилей (HTTP/browser fallback)."""
-
-    def __init__(self):
-        self.db: Session = get_db_session()
-        self.scraper = IAAIScraper()
-
-    def fetch_vehicle_data(self, candidate: VehicleCandidate) -> Optional[VehicleRawSnapshot]:
-        """
-        Захватывает данные для кандидата автомобиля.
-
-        Args:
-            candidate: Кандидат для обработки
-
-        Returns:
-            VehicleRawSnapshot если захват успешен, None если неудача
-        """
-        logger.info(f"Fetching data for candidate {candidate.id}: {candidate.url}")
-
-        try:
-            # Сначала пытаемся HTTP fast-path
-            raw_data = self._try_http_capture(candidate.url)
-            if raw_data:
-                return self._save_snapshot(candidate.id, "http", True, raw_data)
-
-            # Если HTTP не сработал, используем browser fallback
-            logger.info(f"HTTP failed for {candidate.url}, trying browser fallback")
-            raw_data = self._try_browser_capture(candidate.url)
-            if raw_data:
-                return self._save_snapshot(candidate.id, "browser", True, raw_data)
-
-            # Оба метода failed
-            logger.warning(f"Both HTTP and browser capture failed for {candidate.url}")
-            self._save_snapshot(candidate.id, "browser", False, None, "Both capture methods failed")
-            return None
-
-        except Exception as e:
-            error_msg = f"Unexpected error during capture: {str(e)}"
-            logger.error(f"Error fetching {candidate.url}: {error_msg}")
-            self._save_snapshot(candidate.id, "unknown", False, None, error_msg)
-            return None
-
-    def _try_http_capture(self, url: str) -> Optional[str]:
-        """Пытается захватить данные через HTTP."""
-        try:
-            # Используем существующий метод из scraper
-            # Но нам нужно инициализировать scraper с сессией
-            # Пока заглушка - интегрируем позже
-            # Для теста вернем None, чтобы использовать browser
-            return None
-        except Exception as e:
-            logger.debug(f"HTTP capture failed for {url}: {e}")
-            return None
-
-    def _try_browser_capture(self, url: str) -> Optional[str]:
-        """Пытается захватить данные через browser."""
-        try:
-            # Используем scrape_vehicle_detail из scraper
-            with IAAIScraper() as scraper:
-                result = scraper.scrape_vehicle_detail(url)
-                # Возвращаем HTML или JSON данные
-                return result.get("raw_html") or json.dumps(result)
-        except Exception as e:
-            logger.debug(f"Browser capture failed for {url}: {e}")
-            return None
-
-    def _save_snapshot(self, candidate_id: int, method: str, success: bool,
-                      raw_data: Optional[str], error_message: Optional[str] = None) -> VehicleRawSnapshot:
-        """Сохраняет snapshot в базу данных."""
-        snapshot = VehicleRawSnapshot(
-            candidate_id=candidate_id,
-            method=method,
-            success=success,
-            raw_data=raw_data,
-            error_message=error_message
-        )
-        self.db.add(snapshot)
-        self.db.commit()
-        self.db.refresh(snapshot)
-        return snapshot
-
-    def process_pending_candidates(self, limit: int = 10) -> int:
-        """
-        Обрабатывает ожидающих кандидатов.
-
-        Returns:
-            Количество успешно обработанных кандидатов
-        """
-        from .discovery_service import DiscoveryService
-        discovery = DiscoveryService()
-
-        candidates = discovery.get_pending_candidates(limit)
-        processed_count = 0
-
-        for candidate in candidates:
-            discovery.mark_candidate_processing(candidate.id)
-
-            snapshot = self.fetch_vehicle_data(candidate)
-            if snapshot and snapshot.success:
-                discovery.mark_candidate_processed(candidate.id)
-                processed_count += 1
-            else:
-                discovery.mark_candidate_failed(candidate.id)
-
-        return processed_count

iaai_scraper/parsing/mapper.py

@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
 
 
 class CarMapper:
-	# Преобразование данных IAAI в CarRecord.
+	# Маппер IAAI в CarRecord.
 
 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -48,7 +48,7 @@ class CarMapper:
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
 
 	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
-		# Собираем нормализованную DB-модель.
+		# Собираем DB-модель.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
 		core = payload_insights.get("vehicle_core", {})
@@ -77,7 +77,7 @@ class CarMapper:
 		)
 		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
 		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
-		# Пытаемся определить привод из строки двигателя.
+		# Пробуем взять привод из двигателя.
 		if not drive or drive == "NA":
 			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
 			if engine_text:
@@ -144,7 +144,7 @@ class CarMapper:
 
 	@classmethod
 	def _to_money_int(cls, value: Any) -> int | None:
-		# Нормализация стоимости из разных форматов.
+		# Нормализация цены.
 		if value is None:
 			return None
 		if isinstance(value, bool):
@@ -168,14 +168,14 @@ class CarMapper:
 		for number in numbers:
 			clean = number.replace(" ", "")
 			if "," in clean and "." in clean:
-					# Поддержка 1,234.56 и 1.234,56.
+					# Поддержка двух форматов.
 				if clean.rfind(",") > clean.rfind("."):
 					clean = clean.replace(".", "").replace(",", ".")
 				else:
 					clean = clean.replace(",", "")
 			elif "," in clean:
 				parts = clean.split(",")
-				# 123,45 -> 123.45, иначе разделитель тысяч.
+				# Десятичный или тысячный разделитель.
 				if len(parts[-1]) in {1, 2} and len(parts) == 2:
 					clean = clean.replace(",", ".")
 				else:
@@ -214,7 +214,7 @@ class CarMapper:
 
 	@staticmethod
 	def _parse_odometer(value: Any) -> int:
-		# Разбор пробега из строк IAAI.
+		# Разбор пробега.
 		if value is None:
 			return 0
 		text = str(value).strip()
@@ -223,7 +223,7 @@ class CarMapper:
 		lowered = text.lower()
 		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
 			return 0
-		# Извлекаем число из строкового формата одометра.
+		# Ищем число.
 		numbers = re.findall(r"[\d,]+", text)
 		for num_str in numbers:
 			clean = num_str.replace(",", "")
@@ -294,7 +294,7 @@ class CarMapper:
 		empty_default: str | None,
 		fallback: str | None,
 	) -> str | None:
-		# Общий helper для enum-нормализации.
+		# Общая нормализация enum.
 		text = self._as_str(value).lower()
 		if not text:
 			return empty_default
@@ -329,7 +329,7 @@ class CarMapper:
 		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
 
 	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
-		# Для imageKeys берем самый большой размер.
+		# Для imageKeys берём самый большой размер.
 		best_by_key: dict[str, str] = {}
 		key_order: list[str] = []
 		non_keyed: list[str] = []
@@ -375,11 +375,11 @@ class CarMapper:
 
 	@classmethod
 	def _generate_parser_id(cls, origin_id: str) -> str:
-		# Стабильный parser_id по origin_id.
+		# Стабильный parser_id.
 		digest = hashlib.sha256(origin_id.encode()).digest()
 		alphabet = cls._PARSER_ID_ALPHABET
 		base = len(alphabet)
-		num = int.from_bytes(digest[:17], "big")  # 17 байт = 136 бит, хватает на 22 символа
+		num = int.from_bytes(digest[:17], "big")  # Хватает на 22 символа.
 		chars: list[str] = []
 		for _ in range(22):
 			num, idx = divmod(num, base)
@@ -387,7 +387,7 @@ class CarMapper:
 		return "car-" + "".join(chars)
 
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
-		# Формат: iaai:{lot_number} (аналог copart:94323985).
+		# Формат: iaai:{lot_number}.
 		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
 			text = self._as_str(value)
 			if text:

iaai_scraper/parsing/parser.py

@@ -10,9 +10,9 @@ logger = logging.getLogger("iaai_scraper.parsers")
 
 
 class VehicleParser:
-	# Парсер данных страницы автомобиля.
+	# Парсер страницы авто.
 
-	# Регулярные выражения для парсинга и детекции защиты.
+	# Регулярки парсинга и защиты.
 	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
 	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
 	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
@@ -101,11 +101,11 @@ class VehicleParser:
 		max_fields = len(set(self.DOM_LABEL_MAP.values()))
 
 		for i, line in enumerate(lines):
-			# Ранний выход, когда уже нашли все поля.
+			# Ранний выход.
 			if len(result) >= max_fields:
 				break
 
-			# Сценарий 1: "метка: значение" в одной строке.
+			# Метка и значение в одной строке.
 			colon_pos = line.find(":")
 			if colon_pos > 0:
 				label_part = line[:colon_pos].strip().lower()
@@ -116,7 +116,7 @@ class VehicleParser:
 						result[field_name] = value_part
 						continue
 
-			# Сценарий 2: метка и значение на соседних строках.
+			# Метка и значение в соседних строках.
 			clean = line.rstrip(":").strip().lower()
 			clean_alt = clean.rstrip("#").strip()
 			matched_label = None
@@ -164,7 +164,7 @@ class VehicleParser:
 			page_title = title_match.group(1).strip()
 		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
 
-		# Один проход по payload для всех полей.
+		# Один проход по payload.
 		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
 
 		summary: dict[str, Any] = {"source_url": vehicle_url}
@@ -199,7 +199,7 @@ class VehicleParser:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
 				payloads.append(p)
-				# Дополнительный проход по встроенному JSON.
+				# Доп. проход по JSON.
 				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
 				for field, vals in extra.items():
 					if not summary.get(field):
@@ -207,7 +207,7 @@ class VehicleParser:
 						if v:
 							summary[field] = v
 
-		# Передаём image_urls без повторного извлечения.
+		# Передаём готовые image_urls.
 		image_urls = summary.get("image_urls") or []
 		return {
 			"vehicle_summary": summary,
@@ -325,7 +325,7 @@ class VehicleParser:
 		for script_text in scripts:
 			if "{" not in script_text and "[" not in script_text:
 				continue
-			# Пропускаем слишком большие минифицированные блоки.
+			# Пропускаем большие блоки.
 			if len(script_text) > 51_200:
 				continue
 			try:

iaai_scraper/scheduler_service.py

@@ -1,69 +0,0 @@
-import logging
-import time
-from datetime import datetime, timezone, timedelta
-
-from .core.config import settings
-from .discovery_service import DiscoveryService
-from .fetch_service import FetchService
-from .enrichment_service import EnrichmentService
-
-logger = logging.getLogger("iaai_scraper.scheduler_service")
-
-
-class SchedulerService:
-    """Сервис для планирования и координации ingestion pipeline."""
-
-    def __init__(self):
-        self.discovery = DiscoveryService()
-        self.fetch = FetchService()
-        self.enrichment = EnrichmentService()
-
-    def run_full_pipeline(self):
-        """Запускает полный цикл ingestion: discovery -> fetch -> enrichment."""
-        logger.info("Starting full ingestion pipeline")
-
-        try:
-            # 1. Discovery phase
-            logger.info("Phase 1: Discovery")
-            discovered_count = self.discovery.discover_new_vehicles()
-            logger.info(f"Discovered {discovered_count} new candidates")
-
-            # 2. Fetch phase
-            logger.info("Phase 2: Fetch")
-            fetched_count = self.fetch.process_pending_candidates(limit=50)
-            logger.info(f"Successfully fetched {fetched_count} candidates")
-
-            # 3. Enrichment phase
-            logger.info("Phase 3: Enrichment")
-            enriched_count = self.enrichment.process_unparsed_snapshots(limit=50)
-            logger.info(f"Successfully enriched {enriched_count} snapshots")
-
-            logger.info("Ingestion pipeline completed")
-
-        except Exception as e:
-            logger.error(f"Pipeline failed: {e}")
-            raise
-
-    def run_continuous_pipeline(self, interval_minutes: int = 30):
-        """Запускает непрерывный цикл ingestion с интервалом."""
-        logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals")
-
-        while True:
-            try:
-                self.run_full_pipeline()
-            except Exception as e:
-                logger.error(f"Pipeline iteration failed: {e}")
-
-            logger.info(f"Sleeping for {interval_minutes} minutes")
-            time.sleep(interval_minutes * 60)
-
-    def run_targeted_enrichment(self):
-        """Запускает только enrichment для существующих snapshots."""
-        logger.info("Running targeted enrichment")
-        enriched_count = self.enrichment.process_unparsed_snapshots(limit=100)
-        logger.info(f"Enriched {enriched_count} snapshots")
-
-    def cleanup_old_data(self, days_to_keep: int = 30):
-        """Очищает старые данные (опционально)."""
-        # TODO: implement if needed
-        pass

iaai_scraper/storage/db.py

@@ -14,13 +14,6 @@ from .schemas import CarRecord
 logger = logging.getLogger("iaai_scraper.db")
 
 
-def get_db_session() -> Session:
-    """Получить новую сессию базы данных."""
-    settings = Settings()
-    persistence = PersistenceService(settings)
-    return persistence.session_factory()
-
-
 CAR_DB_FIELDS = {
 	col.key for col in Car.__table__.columns
 	if col.key not in ("id",)
@@ -42,11 +35,16 @@ class PersistenceService:
 			engine_kwargs["max_overflow"] = settings.database.max_overflow
 			engine_kwargs["pool_pre_ping"] = True
 			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
+			engine_kwargs["pool_timeout"] = 30
+			# Таймауты запросов и блокировок.
+			engine_kwargs["connect_args"] = {
+				"options": "-c statement_timeout=120000 -c lock_timeout=30000"
+			}
 		self.engine = create_engine(settings.database.url, **engine_kwargs)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
 
 	def create_tables(self) -> None:
-		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
+		# Для SQLite можно create_all.
 		is_sqlite = self.settings.database.url.startswith("sqlite")
 		if not is_sqlite and not self.settings.database.auto_create_tables:
 			return
@@ -112,7 +110,7 @@ class PersistenceService:
 	def get_existing_urls_and_ids(
 		self, origin_urls: list[str], origin_ids: list[str],
 	) -> tuple[set[str], set[str]]:
-		# Загрузка существующих URL и origin_id.
+		# Загрузка URL и origin_id.
 		if not origin_urls and not origin_ids:
 			return set(), set()
 		urls: set[str] = set()
@@ -321,7 +319,7 @@ class PersistenceService:
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def upsert_car(self, record: CarRecord):
-		# Вставка или обновление автомобиля по origin_id/origin_url.
+		# Вставка или обновление авто.
 		payload = self._car_payload(record)
 		images = [image.model_dump(mode="python") for image in record.images]
 		with self.session_scope() as session:
@@ -382,7 +380,7 @@ class PersistenceService:
 		- Один DELETE по car_id IN (...) вместо удаления по одному.
 		- Fallback на по-одному upsert если batch commit упал.
 		"""
-		# ── Дедупликация записей внутри батча ──
+		# Дедупликация батча.
 		seen_ids: dict[str, int] = {}
 		unique_records: list[CarRecord] = []
 		for idx, r in enumerate(records):
@@ -413,20 +411,20 @@ class PersistenceService:
 		images_total = 0
 
 		with self.session_scope() as session:
-			# Получаем существующие записи chunked-запросами.
+			# Загружаем существующие записи.
 			origin_ids = [r.origin_id for r in records if r.origin_id]
 			origin_urls = [r.origin_url for r in records if r.origin_url]
 
 			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
 
-			# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
+			# Предзагружаем изображения.
 			existing_car_ids = set()
 			for record in records:
 				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
 				if car is not None:
 					existing_car_ids.add(int(car.id))
 
-			# Строим маппинг car_id → set(image_urls) для сравнения.
+			# Готовим map car_id -> image_urls.
 			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
 
 			new_cars: list[tuple[Car, list[dict]]] = []
@@ -448,7 +446,7 @@ class PersistenceService:
 					car.last_seen_at = record.last_seen_at
 					updated += 1
 
-					# Проверяем, изменились ли изображения.
+					# Проверяем изменения картинок.
 					new_image_urls = {img.get("fullres_image", "") for img in images}
 					old_image_urls = existing_images_map.get(int(car.id), set())
 					if new_image_urls != old_image_urls:
@@ -456,15 +454,15 @@ class PersistenceService:
 					else:
 						images_total += len(old_image_urls)
 
-			# Один flush для всех вставок.
+			# Один flush.
 			session.flush()
 
-			# Добавляем изображения для новых автомобилей.
+			# Добавляем картинки новым авто.
 			for car, images in new_cars:
 				self._add_images(session, int(car.id), images)
 				images_total += len(images)
 
-			# Массово обновляем изображения только для машин с изменёнными картинками.
+			# Обновляем только изменённые картинки.
 			if update_cars_needing_images:
 				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
 				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
@@ -477,7 +475,7 @@ class PersistenceService:
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
-		# Fallback на поштучный upsert.
+		# Запасной поштучный upsert.
 		inserted = 0
 		updated = 0
 		images_total = 0
@@ -517,72 +515,107 @@ class PersistenceService:
 
 		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
 		что кардинально быстрее при больших объёмах (100K+ URLs).
+		Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент.
 		"""
 		if not active_origin_urls:
 			return 0
 
+		# Нормализация URL.
+		def _norm(url: str) -> str:
+			return url.replace("~", "-")
+
+		normalized_urls = {_norm(u) for u in active_origin_urls}
+
 		is_postgres = "postgresql" in self.settings.database.url
 
 		with self.session_scope() as session:
-			# Страховка от ложного mark_sold при битом/неполном full-scan:
-			# если текущий список активных URL аномально мал относительно уже активных машин в БД,
-			# ничего не помечаем проданным.
-			active_db_count = int(session.execute(
-				select(func.count())
-				.select_from(Car)
-				.where(Car.is_sold == False)  # noqa: E712
-				.where(Car.origin_id.like(f"{lane}:%"))
-			).scalar_one() or 0)
-
-			current_active_count = len(active_origin_urls)
-			if active_db_count >= 1000 and current_active_count < max(500, int(active_db_count * 0.25)):
-				logger.warning(
-					"Skipping mark_sold: suspiciously small active set (%d URLs vs %d active in DB)",
-					current_active_count,
-					active_db_count,
-				)
-				return 0
-
 			if is_postgres:
-				# Создаём временную таблицу с активными URL.
-				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT PRIMARY KEY) ON COMMIT DROP"))
+				# Считаем кандидатов на sold.
+				total_active = session.execute(
+					text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
+				).scalar() or 0
+
+				if total_active == 0:
+					return 0
+
+				# Временная таблица URL.
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
 				session.execute(text("TRUNCATE _active_urls"))
 
-				# Вставляем активные URL чанками через executemany.
-				url_list = list(active_origin_urls)
+				# Вставляем URL чанками.
+				url_list = list(normalized_urls)
 				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
 					chunk = url_list[i:i + _IN_CHUNK_SIZE]
-					session.execute(
-						text("INSERT INTO _active_urls (url) VALUES (:url) ON CONFLICT DO NOTHING"),
-						[{"url": url} for url in chunk],
-					)
+					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
+					params = {f"u{j}": url for j, url in enumerate(chunk)}
+					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
 
-				# Массовая пометка проданных в PostgreSQL.
+				# Индекс для JOIN.
+				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
+
+				# Считаем будущие sold.
+				would_mark = session.execute(text("""
+					SELECT count(*)
+					FROM cars c
+					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+					WHERE a.url IS NULL
+					AND c.is_sold = FALSE
+					AND c.origin_id LIKE 'iaai:%%'
+				""")).scalar() or 0
+
+				# Защита от аномалии.
+				if total_active > 100 and would_mark > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						would_mark, total_active, would_mark / total_active * 100,
+					)
+					return 0
+
+				# Массовая пометка sold.
 				result = session.execute(text("""
 					UPDATE cars
 					SET is_sold = TRUE
 					FROM (
 						SELECT c.id
 						FROM cars c
-						LEFT JOIN _active_urls a ON c.origin_url = a.url
+						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
-						AND c.origin_id LIKE :lane_prefix
+						AND c.origin_id LIKE 'iaai:%%'
 					) sub
 					WHERE cars.id = sub.id
-				"""), {"lane_prefix": f"{lane}:%"})
+				"""))
 				count = result.rowcount or 0
 			else:
 				# Упрощённый путь для SQLite.
 				stmt = (
 					update(Car)
-					.where(Car.origin_url.notin_(active_origin_urls))
 					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like(f"{lane}:%"))
+					.where(Car.origin_id.like("iaai:%"))
 					.values(is_sold=True)
 				)
-				result = session.execute(stmt)
-				count = result.rowcount or 0
+				# Загружаем active URL.
+				all_active = session.execute(
+					select(Car.id, Car.origin_url).where(
+						Car.is_sold == False, Car.origin_id.like("iaai:%")  # noqa: E712
+					)
+				).all()
+				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
+
+				if not mark_ids:
+					return 0
+				total_active = len(all_active)
+				if total_active > 100 and len(mark_ids) > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						len(mark_ids), total_active, len(mark_ids) / total_active * 100,
+					)
+					return 0
+
+				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
+					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
+				count = len(mark_ids)
 
 			if count:
 				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
@@ -600,40 +633,43 @@ class PersistenceService:
 			return {str(row[0]) for row in result if row and row[0]}
 
 	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
-		"""Возвращает все активные (не sold) origin_url для указанного lane/prefix."""
+		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
 		with self.session_scope() as session:
 			result = session.execute(
-				select(Car.origin_url)
-				.where(Car.origin_id.like(f"{prefix}%"))
-				.where(Car.is_sold == False)  # noqa: E712
-				.execution_options(yield_per=10000)
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).execution_options(yield_per=10000)
 			)
 			return {str(row[0]) for row in result if row and row[0]}
 
+	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
+		from sqlalchemy import func as sa_func
+		with self.session_scope() as session:
+			result = session.execute(
+				select(sa_func.count()).select_from(Car).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				)
+			)
+			return int(result.scalar() or 0)
+
 	def get_active_origin_urls_batch_for_refresh(
 		self,
-		*,
 		prefix: str = "iaai:",
 		offset: int = 0,
-		limit: int = 3000,
+		limit: int = 500,
 	) -> list[str]:
-		"""Возвращает батч активных origin_url для циклического hourly refresh."""
-		with self.session_scope() as session:
-			rows = session.execute(
-				select(Car.origin_url)
-				.where(Car.origin_id.like(f"{prefix}%"))
-				.where(Car.is_sold == False)  # noqa: E712
-				.order_by(Car.last_seen_at.asc(), Car.id.asc())
-				.offset(max(0, int(offset)))
-				.limit(max(1, int(limit)))
-			).all()
-			return [str(row[0]) for row in rows if row and row[0]]
+		"""Возвращает батч origin_url активных авто для rolling refresh.
 
-	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
+		"""
 		with self.session_scope() as session:
-			return int(session.execute(
-				select(func.count())
-				.select_from(Car)
-				.where(Car.origin_id.like(f"{prefix}%"))
-				.where(Car.is_sold == False)  # noqa: E712
-			).scalar_one() or 0)
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
+			)
+			return [str(row[0]) for row in result if row and row[0]]

iaai_scraper/storage/models.py

@@ -80,65 +80,3 @@ class SyncRun(Base):
 	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
-
-
-class VehicleCandidate(Base):
-	__tablename__ = "vehicle_candidates"
-	__table_args__ = (
-		Index("ix_vehicle_candidates_url", "url"),
-		Index("ix_vehicle_candidates_status_discovered", "status", "discovered_at"),
-	)
-	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
-	url: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
-	discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
-	status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True)  # pending, processing, processed, failed
-	priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
-	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	raw_snapshots: Mapped[list["VehicleRawSnapshot"]] = relationship("VehicleRawSnapshot", back_populates="candidate", cascade="all, delete-orphan")
-
-
-class VehicleRawSnapshot(Base):
-	__tablename__ = "vehicle_raw_snapshots"
-	__table_args__ = (
-		Index("ix_vehicle_raw_snapshots_candidate_id", "candidate_id"),
-		Index("ix_vehicle_raw_snapshots_captured_at", "captured_at"),
-	)
-	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
-	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
-	candidate: Mapped[VehicleCandidate] = relationship("VehicleCandidate", back_populates="raw_snapshots")
-	captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
-	method: Mapped[str] = mapped_column(String(20), nullable=False)  # http, browser
-	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
-	raw_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # HTML or JSON content
-	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
-	parse_results: Mapped[list["VehicleParseResult"]] = relationship("VehicleParseResult", back_populates="snapshot", cascade="all, delete-orphan")
-
-
-class VehicleParseResult(Base):
-	__tablename__ = "vehicle_parse_results"
-	__table_args__ = (
-		Index("ix_vehicle_parse_results_snapshot_id", "snapshot_id"),
-		Index("ix_vehicle_parse_results_parsed_at", "parsed_at"),
-	)
-	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
-	snapshot_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True)
-	snapshot: Mapped[VehicleRawSnapshot] = relationship("VehicleRawSnapshot", back_populates="parse_results")
-	parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
-	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
-	parsed_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # JSON with parsed vehicle data
-	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
-
-
-class VehicleRetryQueue(Base):
-	__tablename__ = "vehicle_retry_queue"
-	__table_args__ = (
-		Index("ix_vehicle_retry_queue_candidate_id", "candidate_id"),
-		Index("ix_vehicle_retry_queue_retry_at", "retry_at"),
-	)
-	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
-	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
-	reason: Mapped[str] = mapped_column(String(50), nullable=False)  # parse_failed, capture_failed, etc.
-	retry_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
-	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)

iaai_scraper/worker/celery_app.py

@@ -68,7 +68,7 @@ celery_app.conf.update(
     task_track_started=True,
     worker_concurrency=settings.celery.worker_concurrency,
     worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_pool="prefork",
+    worker_pool="solo",
     worker_prefetch_multiplier=1,
     broker_connection_retry_on_startup=True,
     broker_transport_options={
@@ -82,8 +82,11 @@ celery_app.conf.update(
             "task": "iaai_scraper.worker.tasks.sync_listing_task",
             "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
             "args": (),
-            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False},
-            "options": {"queue": "scraping"},
+            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
+            "options": {
+                "queue": "scraping",
+                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
+            },
         }
     },
     task_routes={
@@ -96,8 +99,8 @@ celery_app.autodiscover_tasks(["iaai_scraper.worker"])
 
 @worker_ready.connect
 def _on_worker_ready(**kwargs):
-    """Сразу при старте worker отправляем первую задачу sync_listing,
-    чтобы не ждать час до первого beat-цикла."""
+    """При старте worker отправляем первый sync_listing, если очередь пуста."""
+    redis_client = None
     try:
         redis_client = Redis.from_url(
             settings.redis.url,
@@ -107,10 +110,34 @@ def _on_worker_ready(**kwargs):
             health_check_interval=settings.redis.health_check_interval_seconds,
             retry_on_timeout=True,
         )
+
+        for stale_key in ("iaai:locks:sync_listing",):
+            try:
+                ttl = redis_client.ttl(stale_key)
+                if ttl is not None and ttl != -2:
+                    redis_client.delete(stale_key)
+                    logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+            except Exception:
+                logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
+
+        try:
+            queue_len = int(redis_client.llen("scraping") or 0)
+        except Exception:
+            queue_len = 0
+        if queue_len > 0:
+            logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len)
+            return
+
         should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
     except Exception:
         logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
         return
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
 
     if not should_dispatch:
         logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
@@ -121,4 +148,5 @@ def _on_worker_ready(**kwargs):
         "iaai_scraper.worker.tasks.sync_listing_task",
         kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
         queue="scraping",
+        expires=settings.celery.beat_sync_interval_minutes * 60.0,
     )

iaai_scraper/worker/self_heal.py

@@ -0,0 +1,219 @@
+import json
+import logging
+import os
+import random
+import signal
+import time
+
+from redis import Redis
+
+
+logger = logging.getLogger("iaai_scraper.worker.self_heal")
+
+GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
+SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress"
+SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
+
+
+def _env_bool(name: str, default: bool) -> bool:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	try:
+		return int(value.strip())
+	except Exception:
+		return default
+
+
+def _get_redis() -> Redis:
+	url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0")
+	return Redis.from_url(
+		url,
+		decode_responses=True,
+		socket_connect_timeout=5.0,
+		socket_timeout=10.0,
+		health_check_interval=30,
+		retry_on_timeout=True,
+	)
+
+
+def _safe_int(value: str | None, default: int = 0) -> int:
+	if value is None:
+		return default
+	try:
+		return int(str(value).strip())
+	except Exception:
+		return default
+
+
+def _read_last_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
+	# Это дороже, но выполняется только при отсутствии основного маркера.
+	max_ts = 0
+	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			ts = _safe_int(data.get("ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
+	"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
+	queue_len = _safe_int(redis_client.llen(queue_name), 0)
+	has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0
+	has_task_progress = 0
+	for _ in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		has_task_progress = 1
+		break
+	flags = {
+		"queue_len": queue_len,
+		"has_lock": has_lock,
+		"has_task_progress": has_task_progress,
+	}
+	return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags
+
+
+def _kill_worker_process() -> None:
+	pid_file = "/tmp/celery-worker.pid"
+	pid: int | None = None
+	try:
+		with open(pid_file, "r", encoding="utf-8") as f:
+			pid = int(f.read().strip())
+	except Exception:
+		pid = None
+
+	if not pid:
+		logger.error("Self-heal: failed to read worker pid from %s", pid_file)
+		return
+
+	logger.error("Self-heal: terminating stuck worker process pid=%s", pid)
+	try:
+		os.kill(pid, signal.SIGTERM)
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid)
+		return
+
+	time.sleep(20)
+	try:
+		# Если процесс ещё жив — принудительно убиваем.
+		os.kill(pid, 0)
+		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
+		os.kill(pid, SIGKILL_FALLBACK)
+	except ProcessLookupError:
+		pass
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid)
+
+
+def main() -> None:
+	if not _env_bool("IAAI_SELF_HEAL_ENABLED", True):
+		logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED")
+		return
+
+	queue_name = os.getenv("IAAI_CELERY_QUEUE", "scraping")
+	check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
+	stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720))
+	startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
+	restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
+
+	logger.warning(
+		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
+		queue_name,
+		check_interval,
+		stall_seconds,
+		startup_grace,
+		restart_cooldown,
+	)
+
+	started_at = time.time()
+	redis_client: Redis | None = None
+
+	while True:
+		try:
+			if redis_client is None:
+				redis_client = _get_redis()
+				redis_client.ping()
+
+			has_inflight, inflight = _has_inflight_work(redis_client, queue_name)
+			if not has_inflight:
+				time.sleep(check_interval)
+				continue
+
+			last_progress_ts = _read_last_progress_ts(redis_client)
+			now_ts = int(time.time())
+			age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts))
+
+			if age is None:
+				if now_ts - int(started_at) < startup_grace:
+					time.sleep(check_interval)
+					continue
+				logger.warning(
+					"Self-heal: inflight=%s but no progress timestamp found after startup grace",
+					inflight,
+				)
+				age = stall_seconds + 1
+
+			if age <= stall_seconds:
+				time.sleep(check_interval)
+				continue
+
+			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
+			acquired = bool(
+				redis_client.set(
+					SELF_HEAL_RESTART_LOCK_KEY,
+					str(now_ts),
+					nx=True,
+					ex=restart_cooldown,
+				)
+			)
+			if not acquired:
+				time.sleep(check_interval)
+				continue
+
+			logger.error(
+				"Self-heal: detected global stall (inflight=%s, progress_age=%ss > %ss). Restarting worker process...",
+				inflight,
+				age,
+				stall_seconds,
+			)
+			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
+			# перезапуск был не строго одновременно.
+			time.sleep(random.uniform(0.3, 2.0))
+			_kill_worker_process()
+			# После kill pid1 контейнер будет перезапущен Docker restart-policy.
+			# На случай неуспеха не молотим цикл.
+			time.sleep(check_interval)
+
+		except Exception:
+			logger.exception("Self-heal watchdog iteration failed")
+			redis_client = None
+			time.sleep(check_interval)
+
+
+if __name__ == "__main__":
+	logging.basicConfig(
+		level=os.getenv("IAAI_LOG_LEVEL", "INFO"),
+		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+	)
+	main()

pyproject.toml

@@ -11,7 +11,6 @@ requires-python = ">=3.11"
 dependencies = [
   "alembic>=1.14.0",
   "celery>=5.4.0",
-  "curl-cffi>=0.11.0",
   "fastapi>=0.115.0",
   "playwright>=1.53.0",
   "psycopg2-binary>=2.9.9",
@@ -21,8 +20,6 @@ dependencies = [
   "sqlalchemy>=2.0.32",
   "uvicorn>=0.34.0",
   "urllib3>=2.0.0",
-  "orjson>=3.10.0",
-  "brotli>=1.1.0",
 ]
 
 [project.optional-dependencies]

tests/test_listing.py

@@ -8,83 +8,26 @@ from iaai_scraper.browser.listing import ListingCollector
 
 
 class _FakePage:
-    def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None:
+    def __init__(self, counts: dict[str, int]) -> None:
         self._counts = counts
-        self._attrs = attrs or {}
         self._evaluate_result = False
         self._evaluate_values: list[object] = []
-        self._html = ""
-        self.url = "https://www.iaai.com/Vehiclelisting/Cars"
-        self._current_page_number: int | None = None
-        self._vehicle_hrefs: list[str] = []
-        self._wait_for_function_error: Exception | None = None
-        self._clicks: dict[str, int] = {}
-        self._goto_calls: list[str] = []
 
     class _Locator:
-        def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None:
-            self._page = page
-            self._selector = selector
+        def __init__(self, count_value: int) -> None:
             self._count_value = count_value
 
-        @property
-        def first(self) -> "_FakePage._Locator":
-            return self
-
         def count(self) -> int:
             return self._count_value
 
-        def is_visible(self, timeout: int | None = None) -> bool:
-            _ = timeout
-            return False
+    def locator(self, selector: str) -> "_FakePage._Locator":
+        return _FakePage._Locator(self._counts.get(selector, 0))
 
-        def get_attribute(self, name: str, timeout: int | None = None) -> str | None:
-            _ = timeout
-            return self._page._attrs.get(self._selector, {}).get(name)
-
-        def click(self, timeout: int | None = None) -> None:
-            _ = timeout
-            self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1
-            return None
-
-    class _HtmlLocator:
-        def __init__(self, html: str) -> None:
-            self._html = html
-
-        def inner_html(self, timeout: int | None = None) -> str:
-            _ = timeout
-            return self._html
-
-    def locator(self, selector: str):
-        if selector == "html":
-            return _FakePage._HtmlLocator(self._html)
-        return _FakePage._Locator(self, selector, self._counts.get(selector, 0))
-
-    def evaluate(self, script: str, *args):
-        _ = args
+    def evaluate(self, _script: str):
         if self._evaluate_values:
             return self._evaluate_values.pop(0)
-        if "querySelectorAll" in script and "VehicleDetail" in script:
-            return list(self._vehicle_hrefs)
-        if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script:
-            return self._current_page_number if self._current_page_number is not None else self._evaluate_result
         return self._evaluate_result
 
-    def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
-        _ = selector, timeout
-        return None
-
-    def wait_for_function(self, script: str, timeout: int | None = None) -> None:
-        _ = script, timeout
-        if self._wait_for_function_error is not None:
-            raise self._wait_for_function_error
-        return None
-
-    def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None:
-        _ = wait_until, timeout
-        self._goto_calls.append(url)
-        self.url = url
-
 
 class TestListingUnit(unittest.TestCase):
     def test_pagination_detection_and_page_number(self) -> None:
@@ -120,67 +63,6 @@ class TestListingUnit(unittest.TestCase):
             ],
         )
 
-    def test_has_next_page_from_html(self) -> None:
-        collector = ListingCollector(Settings(), HumanPacer(Settings()))
-        html = '<a class="pagination-next" href="/Vehiclelisting/Cars?page=43">Next</a>'
-        self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42))
-        self.assertFalse(collector._has_next_page_from_html("<div>done</div>", current_page_number=42))
-
-    def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None:
-        collector = ListingCollector(Settings(), HumanPacer(Settings()))
-
-        url = collector._build_listing_page_url(
-            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar",
-            44,
-        )
-
-        self.assertEqual(
-            url,
-            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44",
-        )
-
-    def test_collect_current_page_uses_html_first(self) -> None:
-        collector = ListingCollector(Settings(), HumanPacer(Settings()))
-        page = _FakePage({})
-        page._html = '<a href="/VehicleDetail/555~US">Car 555</a><a class="pagination-next" href="/Vehiclelisting/Cars?page=2">Next</a>'
-
-        result = collector.collect_current_page(page, page_number=1)
-
-        self.assertEqual(len(result.vehicle_links), 1)
-        self.assertEqual(result.vehicle_links[0].lot_number, "555")
-        self.assertTrue(result.next_page_detected)
-
-    def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None:
-        page = _FakePage({})
-        page._wait_for_function_error = RuntimeError("same content")
-        page._current_page_number = 41
-        page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
-
-        self.assertFalse(
-            ListingCollector._wait_for_navigation_result(
-                page,
-                "/VehicleDetail/111~US",
-                41,
-                old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"),
-            )
-        )
-
-    def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None:
-        collector = ListingCollector(Settings(), HumanPacer(Settings()))
-        page = _FakePage(
-            {ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1},
-            attrs={
-                "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"},
-            },
-        )
-        page._wait_for_function_error = RuntimeError("same content")
-        page._current_page_number = 41
-        page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
-
-        self.assertFalse(collector.go_to_next_page(page, expected_page_number=41))
-        self.assertGreaterEqual(len(page._goto_calls), 1)
-        self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0)
-
 
 if __name__ == "__main__":
     unittest.main()

tests/test_resilience.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import unittest
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.scraper import IAAIScraper
+from iaai_scraper.core.config import Settings
+from iaai_scraper.worker import tasks
+
+
+class TestResilience(unittest.TestCase):
+	def _make_scraper(self) -> IAAIScraper:
+		s = Settings()
+		s.log_level = "CRITICAL"
+		s.database.url = "sqlite://"
+		return IAAIScraper(s)
+
+	def test_update_task_progress_updates_global_marker(self) -> None:
+		redis_client = MagicMock()
+		pipe = MagicMock()
+		redis_client.pipeline.return_value = pipe
+
+		tasks._update_task_progress(
+			redis_client,
+			task_id="task-abc",
+			stage="batch_upserted",
+			ttl_seconds=180,
+			cars_upserted=10,
+		)
+
+		redis_client.pipeline.assert_called_once()
+		self.assertEqual(pipe.set.call_count, 2)
+		first_call = pipe.set.call_args_list[0]
+		second_call = pipe.set.call_args_list[1]
+
+		self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc"))
+		self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY)
+		pipe.execute.assert_called_once()
+
+	def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None:
+		scraper = self._make_scraper()
+		scraper.settings.celery.batch_size = 50
+
+		page = MagicMock()
+		empty_page_result = SimpleNamespace(
+			page_number=1,
+			vehicle_links=[],
+			next_page_detected=True,
+		)
+
+		scraper._open_listing_for_stream = MagicMock(return_value=(
+			page,
+			{"make": None, "model": None, "year_min": None, "year_max": None},
+		))
+		scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result)
+		scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, []))
+		scraper.sync_batch = MagicMock()
+
+		result = scraper._sync_listing_streaming(
+			make=None,
+			model=None,
+			lane="iaai_cars",
+			limit=None,
+			effective_only_new=False,
+			started_at=0.0,
+			listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
+		)
+
+		self.assertEqual(result["total"], 0)
+		self.assertEqual(result["cars_upserted"], 0)
+		self.assertEqual(result["cars_failed"], 0)
+		self.assertEqual(result["listing"]["pages_collected"], 1)
+		scraper._recover_empty_listing_page.assert_called_once()
+		scraper.sync_batch.assert_not_called()
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_scraper.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
 import unittest
+from concurrent.futures import TimeoutError as FuturesTimeoutError
 from types import SimpleNamespace
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 from iaai_scraper.core.config import Settings
 from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
-from iaai_scraper.core.exceptions import ListingResumeError
-from iaai_scraper.discovery import SitemapDiscoveryError
 from iaai_scraper.scraper import IAAIScraper
 from iaai_scraper.storage.schemas import CarRecord
 
@@ -87,51 +86,6 @@ class TestScraperSync(unittest.TestCase):
         scraper2._sync_listing_streaming.assert_called_once()
         scraper2.collect_listing.assert_not_called()
 
-    def test_full_scan_uses_sitemap_discovery_path(self) -> None:
-        scraper = self._make_scraper()
-        scraper.settings.discovery.mode = "listing"
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=88)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
-        scraper._sync_listing_via_sitemap = MagicMock(return_value={
-            "listing": {
-                "vehicles_collected": 123,
-                "early_stopped": False,
-                "truncated_by_time_budget": False,
-                "pagination_interrupted": False,
-                "source": "sitemap",
-            },
-            "total": 123,
-            "skipped_existing": 0,
-            "cars_upserted": 120,
-            "cars_failed": 3,
-            "images_upserted": 500,
-            "failures": [{"vehicle_url": "v", "error": "e"}],
-            "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"},
-        })
-        scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used"))
-
-        result = scraper.sync_listing()
-
-        scraper._sync_listing_via_sitemap.assert_called_once()
-        scraper._sync_listing_streaming.assert_not_called()
-        self.assertEqual(result["cars_upserted"], 120)
-        self.assertTrue(result["full_scan_completed"])
-
-    def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None:
-        scraper = self._make_scraper()
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=89)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
-        scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down"))
-        result = scraper.sync_listing()
-
-        scraper._sync_listing_via_sitemap.assert_called_once()
-        self.assertEqual(result["status"], "failed")
-        self.assertEqual(result["cars_upserted"], 0)
-
     def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
         scraper = self._make_scraper()
         scraper.persistence.create_tables = MagicMock()
@@ -315,125 +269,50 @@ class TestScraperSync(unittest.TestCase):
         self.assertEqual(result["cars_upserted"], 1)
         self.assertEqual(result["total"], 1)
 
-    def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None:
+    def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
         scraper = self._make_scraper()
-        scraper.settings.celery.batch_size = 1000
-
-        page = MagicMock()
-        page_result = SimpleNamespace(
-            page_number=1,
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
-            next_page_detected=True,
-        )
-
-        scraper._get_page_with_warmup = MagicMock(return_value=page)
-        scraper.listing_collector.open_cars_listing = MagicMock()
-        scraper.listing_collector.apply_filters = MagicMock(return_value={
-            "make": None,
-            "model": None,
-            "year_min": None,
-            "year_max": None,
-        })
-        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
-        scraper.listing_collector.go_to_next_page = MagicMock(return_value=False)
-        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
-        scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed"))
-        scraper.sync_batch = MagicMock(return_value={
-            "cars_upserted": 0,
-            "cars_failed": 0,
+        scraper.persistence.upsert_cars_batch = MagicMock(return_value={
+            "inserted": 1,
+            "updated": 0,
             "images_upserted": 0,
-            "failures": [],
         })
 
-        result = scraper._sync_listing_streaming(
-            make=None,
-            model=None,
-            lane="iaai_cars",
-            limit=None,
-            effective_only_new=False,
-            started_at=0.0,
-            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
-        )
+        urls = [
+            "https://www.iaai.com/VehicleDetail/111~US",
+            "https://www.iaai.com/VehicleDetail/222~US",
+            "https://www.iaai.com/VehicleDetail/333~US",
+        ]
+        first_record = CarRecord.model_validate(make_db_record("111"))
 
-        self.assertTrue(result["listing"]["pagination_interrupted"])
+        class _FakeExecutor:
+            def __init__(self, *args, **kwargs):
+                pass
 
-    def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None:
-        scraper = self._make_scraper()
-        scraper.settings.celery.batch_size = 1000
+            def __enter__(self):
+                return self
 
-        page = MagicMock()
-        first_page = SimpleNamespace(
-            page_number=1,
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")],
-            next_page_detected=False,
-        )
-        second_page = SimpleNamespace(
-            page_number=2,
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")],
-            next_page_detected=False,
-        )
+            def __exit__(self, exc_type, exc, tb):
+                return False
 
-        scraper._get_page_with_warmup = MagicMock(return_value=page)
-        scraper.listing_collector.open_cars_listing = MagicMock()
-        scraper.listing_collector.apply_filters = MagicMock(return_value={
-            "make": None,
-            "model": None,
-            "year_min": None,
-            "year_max": None,
-        })
-        scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page])
-        scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False])
-        scraper._extract_page_urls = MagicMock(side_effect=[
-            ["https://www.iaai.com/VehicleDetail/111~US"],
-            ["https://www.iaai.com/VehicleDetail/222~US"],
-        ])
-        scraper.sync_batch = MagicMock(return_value={
-            "cars_upserted": 2,
-            "cars_failed": 0,
-            "images_upserted": 0,
-            "failures": [],
-        })
+            def map(self, fn, iterable, timeout=None):  # noqa: ARG002
+                yield 0, first_record
+                raise FuturesTimeoutError()
 
-        result = scraper._sync_listing_streaming(
-            make=None,
-            model=None,
-            lane="iaai_cars",
-            limit=None,
-            effective_only_new=False,
-            started_at=0.0,
-            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
-        )
+        with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
+             patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
+             patch.object(scraper, "_browser_fallback_parallel", return_value={
+                 "records": [],
+                 "failures": [],
+                 "cars_failed": 0,
+                 "protection_events": 0,
+             }) as fallback_mock:
+            result = scraper.sync_batch(urls)
 
-        self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2)
-        scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2)
-        self.assertEqual(result["listing"]["pages_collected"], 2)
-
-    def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None:
-        scraper = self._make_scraper()
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=77)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
-        scraper._sync_listing_streaming = MagicMock(return_value={
-            "listing": {
-                "vehicles_collected": 10,
-                "early_stopped": False,
-                "truncated_by_time_budget": False,
-                "pagination_interrupted": True,
-            },
-            "total": 10,
-            "skipped_existing": 0,
-            "cars_upserted": 10,
-            "cars_failed": 0,
-            "images_upserted": 0,
-            "failures": [],
-            "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"},
-        })
-
-        result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA")
-
-        self.assertTrue(result["full_scan_completed"] is False)
-        scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
+        self.assertEqual(result["cars_upserted"], 1)
+        fallback_urls = fallback_mock.call_args.args[0]
+        self.assertEqual(len(fallback_urls), 2)
+        self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
+        self.assertNotIn(urls[0], {u for u, _ in fallback_urls})
 
 
 if __name__ == "__main__":

tests/test_self_heal.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.worker import self_heal
+
+
+class TestSelfHeal(unittest.TestCase):
+	def test_read_last_progress_ts_uses_global_key(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.return_value = "1776800000"
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertEqual(ts, 1776800000)
+		redis_client.scan_iter.assert_not_called()
+
+	def test_read_last_progress_ts_fallbacks_to_task_progress_keys(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.side_effect = lambda key: {
+			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
+			"iaai:state:task_progress:a": '{"ts": 100}',
+			"iaai:state:task_progress:b": '{"ts": 250}',
+			"iaai:state:task_progress:c": '{"ts": 150}',
+		}.get(key)
+		redis_client.scan_iter.return_value = [
+			"iaai:state:task_progress:a",
+			"iaai:state:task_progress:b",
+			"iaai:state:task_progress:c",
+		]
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertEqual(ts, 250)
+
+	def test_read_last_progress_ts_ignores_broken_payloads(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.side_effect = lambda key: {
+			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
+			"iaai:state:task_progress:a": "{bad-json}",
+			"iaai:state:task_progress:b": '{"foo": "bar"}',
+		}.get(key)
+		redis_client.scan_iter.return_value = [
+			"iaai:state:task_progress:a",
+			"iaai:state:task_progress:b",
+		]
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertIsNone(ts)
+
+	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("iaai_scraper.worker.self_heal.os.kill")
+	@patch("builtins.open")
+	def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
+		open_mock.return_value.__enter__.return_value.read.return_value = "123"
+		# SIGTERM -> process alive check (pid,0) -> SIGKILL
+		kill_mock.side_effect = [None, None, None]
+
+		self_heal._kill_worker_process()
+
+		self.assertEqual(kill_mock.call_args_list[0].args[0], 123)
+		self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
+		self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
+
+	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("iaai_scraper.worker.self_heal.os.kill")
+	@patch("builtins.open")
+	def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
+		open_mock.return_value.__enter__.return_value.read.return_value = "123"
+		kill_mock.side_effect = [None, ProcessLookupError()]
+
+		self_heal._kill_worker_process()
+
+		# Только SIGTERM и проверка существования процесса.
+		self.assertEqual(len(kill_mock.call_args_list), 2)
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_sitemap_discovery.py

@@ -1,217 +0,0 @@
-from __future__ import annotations
-
-import unittest
-from unittest.mock import patch
-
-from iaai_scraper.core.config import Settings
-from iaai_scraper.discovery.sitemap import (
-    SitemapBlockedError,
-    SitemapDiscoveryError,
-    SitemapFetchResult,
-    _filter_vehicle_urls,
-    discover_vehicle_urls_from_sitemap,
-    discover_vehicle_urls_from_sitemap_with_stats,
-)
-
-
-class TestSitemapDiscovery(unittest.TestCase):
-    @staticmethod
-    def _fetch_result(url: str, payload: bytes, *, source: str = "curl_cffi") -> SitemapFetchResult:
-        return SitemapFetchResult(url=url, payload=payload, source=source)
-
-    def test_filter_vehicle_urls_dedupes_and_normalizes(self) -> None:
-        urls = _filter_vehicle_urls([
-            "https://www.iaai.com/VehicleDetail/111~US?foo=1",
-            "https://www.iaai.com/VehicleDetail/111~US?bar=2",
-            "https://www.iaai.com/VehicleDetail/222~US",
-            "https://www.iaai.com/about",
-        ])
-
-        self.assertEqual(
-            urls,
-            [
-                "https://www.iaai.com/VehicleDetail/111~US",
-                "https://www.iaai.com/VehicleDetail/222~US",
-            ],
-        )
-
-    def test_discover_vehicle_urls_from_sitemap(self) -> None:
-        index_xml = b"""
-        <sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
-          <sitemap><loc>https://www.iaai.com/sitemap-a.xml</loc></sitemap>
-          <sitemap><loc>https://www.iaai.com/sitemap-b.xml</loc></sitemap>
-                    <sitemap><loc>https://www.iaai.com/sitemapauctions1.xml</loc></sitemap>
-        </sitemapindex>
-        """
-        sitemap_a = b"""
-        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
-          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
-          <url><loc>https://www.iaai.com/VehicleDetail/222~US?x=1</loc></url>
-        </urlset>
-        """
-        sitemap_b = b"""
-        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
-          <url><loc>https://www.iaai.com/VehicleDetail/222~US?y=2</loc></url>
-          <url><loc>https://www.iaai.com/VehicleDetail/333~US</loc></url>
-        </urlset>
-        """
-
-        def _fake_fetch(url: str) -> SitemapFetchResult:
-            if url.endswith("sitemap_index.xml"):
-                return self._fetch_result(url, index_xml)
-            if url.endswith("sitemap-a.xml"):
-                return self._fetch_result(url, sitemap_a)
-            if url.endswith("sitemap-b.xml"):
-                return self._fetch_result(url, sitemap_b)
-            raise AssertionError(f"unexpected url: {url}")
-
-        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
-            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
-
-        self.assertEqual(
-            result,
-            [
-                "https://www.iaai.com/VehicleDetail/111~US",
-                "https://www.iaai.com/VehicleDetail/222~US",
-                "https://www.iaai.com/VehicleDetail/333~US",
-            ],
-        )
-
-    def test_discover_vehicle_urls_raises_on_empty_index(self) -> None:
-        empty_index = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
-        with patch(
-            "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
-            return_value=self._fetch_result("https://www.iaai.com/sitemap_index.xml", empty_index),
-        ):
-            with self.assertRaises(SitemapDiscoveryError):
-                discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
-
-    def test_discover_vehicle_urls_skips_malformed_and_non_vehicle_sitemaps(self) -> None:
-        sitemap_vehicle = b"""
-        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
-            <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
-        </urlset>
-        """
-        sitemap_non_vehicle = b"""
-        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
-            <url><loc>https://www.iaai.com/SalesList/111~US/04222026</loc></url>
-        </urlset>
-        """
-        index_xml = b"""
-        <sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
-            <sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>
-            <sitemap><loc>https://www.iaai.com/sitemapbranches1.xml</loc></sitemap>
-            <sitemap><loc>https://www.iaai.com/sitemap2.xml</loc></sitemap>
-            <sitemap><loc>https://www.iaai.com/sitemap3.xml</loc></sitemap>
-        </sitemapindex>
-        """
-
-        def _fake_fetch(url: str) -> SitemapFetchResult:
-            if url.endswith("sitemap_index.xml"):
-                return self._fetch_result(url, index_xml)
-            if url.endswith("sitemap1.xml"):
-                return self._fetch_result(url, sitemap_vehicle)
-            if url.endswith("sitemap2.xml"):
-                return self._fetch_result(url, sitemap_non_vehicle)
-            if url.endswith("sitemap3.xml"):
-                raise SitemapDiscoveryError("broken sitemap")
-            raise AssertionError(f"unexpected url: {url}")
-
-        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
-            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
-
-        self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
-
-    def test_discover_vehicle_urls_falls_back_to_regex_loc_extraction(self) -> None:
-        malformed_index = (
-            b"<sitemapindex>"
-            b"<sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>"
-            b"<broken"
-        )
-        sitemap_vehicle = b"""
-        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
-          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
-        </urlset>
-        """
-
-        def _fake_fetch(url: str) -> SitemapFetchResult:
-            if url.endswith("sitemap_index.xml"):
-                return self._fetch_result(url, malformed_index)
-            if url.endswith("sitemap1.xml"):
-                return self._fetch_result(url, sitemap_vehicle)
-            raise AssertionError(f"unexpected url: {url}")
-
-        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
-            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
-
-        self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
-
-    def test_discovery_uses_direct_probe_when_index_has_no_urls(self) -> None:
-        index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
-        sitemap_one = b"""
-        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
-          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
-        </urlset>
-        """
-
-        settings = Settings()
-        settings.discovery.sitemap_direct_probe_limit = 2
-        settings.discovery.sitemap_direct_probe_stop_after_misses = 1
-
-        def _fake_fetch(url: str) -> SitemapFetchResult:
-            if url.endswith("sitemap_index.xml"):
-                return self._fetch_result(url, index_xml)
-            if url.endswith("sitemap1.xml"):
-                return self._fetch_result(url, sitemap_one)
-            raise SitemapDiscoveryError("not found")
-
-        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
-            result = discover_vehicle_urls_from_sitemap_with_stats(
-                "https://www.iaai.com/sitemap_index.xml",
-                settings=settings,
-            )
-
-        self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
-        self.assertEqual(result.stats.direct_probe_hits, 1)
-
-    def test_discovery_stats_report_direct_probe_hits(self) -> None:
-        index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
-        sitemap1 = b"""
-        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
-          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
-        </urlset>
-        """
-
-        def _fake_fetch(url: str):
-            if url.endswith("sitemap_index.xml"):
-                return self._fetch_result(url, index_xml)
-            if url.endswith("sitemap1.xml"):
-                return self._fetch_result(url, sitemap1)
-            raise SitemapDiscoveryError("not found")
-
-        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
-            result = discover_vehicle_urls_from_sitemap_with_stats("https://www.iaai.com/sitemap_index.xml")
-
-        self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
-        self.assertEqual(result.stats.transport, "curl_cffi")
-        self.assertEqual(result.stats.direct_probe_hits, 1)
-        self.assertGreaterEqual(result.stats.direct_probe_misses, 1)
-
-    def test_block_page_raises_discovery_error(self) -> None:
-        settings = Settings()
-        settings.discovery.sitemap_direct_probe_limit = 1
-        settings.discovery.sitemap_direct_probe_stop_after_misses = 1
-
-        with patch(
-            "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
-            side_effect=SitemapBlockedError("Anti-bot page returned for https://www.iaai.com/sitemap_index.xml"),
-        ):
-            with self.assertRaises(SitemapDiscoveryError):
-                discover_vehicle_urls_from_sitemap(
-                    "https://www.iaai.com/sitemap_index.xml",
-                    settings=settings,
-                )
-
-
-if __name__ == "__main__":
-    unittest.main()

tests/test_worker_tasks.py

@@ -1,215 +1,15 @@
 from __future__ import annotations
 
+import json
+from dataclasses import replace
 import unittest
 from unittest.mock import MagicMock, patch
 
 from iaai_scraper.worker import tasks
-
-
-def make_settings_stub(*, parallel_segments: bool = False) -> MagicMock:
-    settings_stub = MagicMock()
-    settings_stub.celery.parallel_segments = parallel_segments
-    settings_stub.celery.task_soft_time_limit = 3300
-    settings_stub.celery.task_time_limit = 3600
-    settings_stub.celery.task_stall_timeout_seconds = 600
-    settings_stub.listing.listing_segments_json = "ignored"
-    settings_stub.discovery.mode = "listing"
-    settings_stub.discovery.hourly_mode = "rolling_refresh"
-    return settings_stub
+from iaai_scraper.core.config import settings as base_settings
 
 
 class TestWorkerTaskLockHelpers(unittest.TestCase):
-    def test_sync_listing_task_uses_hourly_sitemap_even_when_mode_is_not_sitemap(self) -> None:
-        settings_stub = make_settings_stub(parallel_segments=False)
-        settings_stub.discovery.mode = "listing"
-        settings_stub.discovery.hourly_mode = "rolling_refresh"
-
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "Settings", return_value=settings_stub), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
-                 "status": "success",
-                 "run_id": None,
-                 "cars_upserted": 25,
-                 "cars_failed": 0,
-                 "images_upserted": 50,
-                 "skipped_existing": 100,
-                 "elapsed_seconds": None,
-                 "failures": [],
-                 "hourly_mode": "sitemap_rolling_refresh",
-                 "discovered_urls": 105,
-                 "new_urls": 5,
-                 "refresh_urls": 20,
-                 "sold_marked": 2,
-             }) as hourly_sync, \
-             patch.object(tasks.sync_listing_task, "update_state"):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            tasks.sync_listing_task.push_request(id="task-hourly-force-sitemap")
-            try:
-                result = tasks.sync_listing_task.run()
-            finally:
-                tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
-        hourly_sync.assert_called_once()
-        release_lock.assert_called_once()
-
-    def test_sync_listing_task_uses_hourly_sitemap_rolling_refresh_after_bootstrap(self) -> None:
-        settings_stub = make_settings_stub(parallel_segments=False)
-        settings_stub.discovery.mode = "sitemap"
-        settings_stub.discovery.hourly_mode = "rolling_refresh"
-
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "Settings", return_value=settings_stub), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
-                 "status": "success",
-                 "run_id": None,
-                 "cars_upserted": 25,
-                 "cars_failed": 0,
-                 "images_upserted": 50,
-                 "skipped_existing": 100,
-                 "elapsed_seconds": None,
-                 "failures": [],
-                 "hourly_mode": "sitemap_rolling_refresh",
-                 "discovered_urls": 105,
-                 "new_urls": 5,
-                 "refresh_urls": 20,
-                 "sold_marked": 2,
-             }) as hourly_sync, \
-             patch.object(tasks.sync_listing_task, "update_state"):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            tasks.sync_listing_task.push_request(id="task-hourly")
-            try:
-                result = tasks.sync_listing_task.run()
-            finally:
-                tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
-        hourly_sync.assert_called_once()
-        release_lock.assert_called_once()
-
-    def test_sync_listing_task_can_use_hourly_sitemap_diff_when_configured(self) -> None:
-        settings_stub = make_settings_stub(parallel_segments=False)
-        settings_stub.discovery.mode = "sitemap"
-        settings_stub.discovery.hourly_mode = "diff"
-
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "Settings", return_value=settings_stub), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=True), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_hourly_sitemap_diff_sync", return_value={
-                 "status": "success",
-                 "run_id": None,
-                 "cars_upserted": 5,
-                 "cars_failed": 0,
-                 "images_upserted": 10,
-                 "skipped_existing": 100,
-                 "elapsed_seconds": None,
-                 "failures": [],
-                 "hourly_mode": "sitemap_diff",
-                 "discovered_urls": 105,
-                 "new_urls": 5,
-                 "sold_marked": 2,
-             }) as hourly_sync, \
-             patch.object(tasks.sync_listing_task, "update_state"):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            tasks.sync_listing_task.push_request(id="task-hourly-diff")
-            try:
-                result = tasks.sync_listing_task.run()
-            finally:
-                tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(result["hourly_mode"], "sitemap_diff")
-        hourly_sync.assert_called_once()
-        release_lock.assert_called_once()
-
-    def test_sync_listing_task_forces_sitemap_full_scan_in_bootstrap(self) -> None:
-        settings_stub = make_settings_stub(parallel_segments=False)
-        settings_stub.discovery.mode = "listing"
-
-        with patch.object(tasks, "_get_persistence") as get_persistence, \
-             patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "Settings", return_value=settings_stub), \
-             patch.object(tasks, "_acquire_lock", return_value=True), \
-             patch.object(tasks, "_is_full_scan_done", return_value=False), \
-             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
-             patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[{"make": "HONDA"}]):
-            get_persistence.return_value = MagicMock()
-            redis_client = MagicMock()
-            redis_client.get.return_value = None
-            get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock())
-
-            sync_listing_mock = MagicMock(return_value={
-                "run_id": 99,
-                "status": "success",
-                "full_scan_completed": True,
-                "cars_upserted": 10,
-                "cars_failed": 0,
-                "images_upserted": 20,
-                "skipped_existing": 0,
-                "elapsed_seconds": 1.0,
-                "failures": [],
-            })
-            sync_listing_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
-            scraper = MagicMock()
-            scraper.sync_listing = sync_listing_mock
-            scraper.sync_listing_segmented = sync_listing_segmented_mock
-            scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value = scraper
-            scraper_ctx.__exit__.return_value = None
-
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
-                tasks.sync_listing_task.push_request(id="task-bootstrap-sitemap")
-                try:
-                    result = tasks.sync_listing_task.run()
-                finally:
-                    tasks.sync_listing_task.pop_request()
-
-        self.assertEqual(result["status"], "success")
-        sync_listing_mock.assert_called_once_with(
-            make=None,
-            model=None,
-            lane="iaai_cars",
-            limit=None,
-            only_new=False,
-        )
-        sync_listing_segmented_mock.assert_not_called()
-        release_lock.assert_called_once()
-
     def test_lock_acquire_refresh_release(self) -> None:
         redis_client = MagicMock()
 
@@ -378,17 +178,15 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         self.assertIsNone(tasks._load_last_completed_segment(redis_client))
         redis_client.delete.assert_not_called()
 
-    def test_sync_listing_bootstrap_prefers_sitemap_over_segment_resume(self) -> None:
+    def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
         segments = [
             {"make": "ACURA"},
             {"make": "AUDI"},
             {"make": "BMW"},
             {"make": "EAGLE"},
         ]
-        settings_stub = make_settings_stub()
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "Settings", return_value=settings_stub), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -404,15 +202,13 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             get_redis.return_value = redis_client
             start_heartbeat.return_value = (MagicMock(), MagicMock())
 
-            sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
-            sync_listing_mock = MagicMock(return_value={
+            sync_segmented_mock = MagicMock(return_value={
                 "run_id": 11, "status": "success", "full_scan_completed": True,
                 "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
                 "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
             })
             scraper_ctx = MagicMock()
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
-            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
             scraper_ctx.__exit__.return_value = None
 
             with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
@@ -423,42 +219,21 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
                     tasks.sync_listing_task.pop_request()
 
         self.assertEqual(result["status"], "success")
-        sync_listing_mock.assert_called_once_with(
-            make=None,
-            model=None,
-            lane="iaai_cars",
-            limit=None,
-            only_new=False,
-        )
-        sync_segmented_mock.assert_not_called()
+        # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
         release_lock.assert_called_once()
 
-    def test_sync_listing_hourly_path_ignores_checkpoint_after_full_scan_completed(self) -> None:
+    def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
         segments = [{"make": "ACURA"}, {"make": "AUDI"}]
-        settings_stub = make_settings_stub()
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "Settings", return_value=settings_stub), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=True), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
-             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
-                 "status": "success",
-                 "run_id": None,
-                 "cars_upserted": 1,
-                 "cars_failed": 0,
-                 "images_upserted": 0,
-                 "skipped_existing": 0,
-                 "elapsed_seconds": None,
-                 "failures": [],
-                 "hourly_mode": "sitemap_rolling_refresh",
-                 "discovered_urls": 10,
-                 "new_urls": 1,
-                 "refresh_urls": 1,
-                 "sold_marked": 0,
-             }) as hourly_sync, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
              patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
@@ -470,24 +245,32 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             get_redis.return_value = redis_client
             start_heartbeat.return_value = (MagicMock(), MagicMock())
 
-            tasks.sync_listing_task.push_request(id="task-792")
-            try:
-                result = tasks.sync_listing_task.run()
-            finally:
-                tasks.sync_listing_task.pop_request()
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 14, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-792")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
 
         self.assertEqual(result["status"], "success")
-        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
-        hourly_sync.assert_called_once()
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
+        self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
         clear_checkpoint.assert_called()
         release_lock.assert_called_once()
 
-    def test_sync_listing_bootstrap_ignores_beyond_segment_checkpoint(self) -> None:
+    def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
         segments = [{"make": "ACURA"}, {"make": "AUDI"}]
-        settings_stub = make_settings_stub()
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
-             patch.object(tasks, "Settings", return_value=settings_stub), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -503,15 +286,13 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             get_redis.return_value = redis_client
             start_heartbeat.return_value = (MagicMock(), MagicMock())
 
-            sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
-            sync_listing_mock = MagicMock(return_value={
+            sync_segmented_mock = MagicMock(return_value={
                 "run_id": 15, "status": "success", "full_scan_completed": True,
                 "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
                 "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
             })
             scraper_ctx = MagicMock()
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
-            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
             scraper_ctx.__exit__.return_value = None
 
             with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
@@ -522,14 +303,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
                     tasks.sync_listing_task.pop_request()
 
         self.assertEqual(result["status"], "success")
-        sync_listing_mock.assert_called_once_with(
-            make=None,
-            model=None,
-            lane="iaai_cars",
-            limit=None,
-            only_new=False,
-        )
-        sync_segmented_mock.assert_not_called()
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
         release_lock.assert_called_once()
 
     def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
@@ -606,38 +380,28 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         self.assertTrue(should_enqueue)
 
     def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
-        settings_stub = make_settings_stub(parallel_segments=False)
         with patch.object(tasks, "_get_persistence") as get_persistence, \
-            patch.object(tasks, "_get_redis") as get_redis, \
-            patch.object(tasks, "Settings", return_value=settings_stub), \
-            patch.object(tasks, "_acquire_lock", return_value=True), \
-            patch.object(tasks, "_is_full_scan_done", return_value=False), \
-            patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
-            patch.object(tasks, "_release_lock_if_owner") as release_lock, \
-            patch.object(tasks, "_try_set_followup_pending", return_value=True), \
-            patch.object(
-                tasks,
-                "_bump_bootstrap_failure_streak",
-                return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False),
-            ) as bump_streak, \
-            patch.object(tasks, "_clear_followup_pending") as clear_pending, \
-            patch.object(tasks.sync_listing_task, "update_state"), \
-            patch.object(
-                tasks,
-                "_run_browser_job",
-                return_value={
-                    "run_id": 99,
-                    "status": "failed",
-                    "full_scan_completed": False,
-                    "cars_upserted": 0,
-                    "cars_failed": 0,
-                    "images_upserted": 0,
-                    "skipped_existing": 0,
-                    "elapsed_seconds": 1.0,
-                    "failures": [{"vehicle_url": "listing", "error": "bad resume"}],
-                    "listing": {"vehicles_collected": 0},
-                },
-            ):
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+             patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
+             patch.object(tasks, "_clear_followup_pending") as clear_pending, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 99,
+                 "status": "failed",
+                 "full_scan_completed": False,
+                 "cars_upserted": 0,
+                 "cars_failed": 0,
+                 "images_upserted": 0,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 1.0,
+                 "failures": [{"vehicle_url": "listing", "error": "bad resume"}],
+                 "listing": {"vehicles_collected": 0},
+             }):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             redis_client.get.return_value = None
@@ -656,6 +420,177 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         clear_pending.assert_called()
         task_app.send_task.assert_not_called()
 
+    def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-soft-timeout")
+                try:
+                    result = tasks.sync_listing_task.run(make="Toyota")
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "timed_out")
+        set_pending.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+             patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 101,
+                 "status": "partial_success",
+                 "full_scan_completed": False,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 1,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 1.0,
+                 "failures": [],
+                 "listing": {"vehicles_collected": 2},
+             }):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-breaker-stop")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "partial_success")
+        # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly.
+        self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list))
+        self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list))
+        clear_checkpoint.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job") as run_job, \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            run_job.return_value = {
+                "run_id": 17,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            }
+
+            tasks.sync_listing_task.push_request(id="task-always-full")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        set_full_scan_done.assert_called_with(redis_client, False)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None:
+        segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 18,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-always-full-resume")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
+        release_lock.assert_called_once()
+
 
 if __name__ == "__main__":
     unittest.main()