fix: remove duplicate index=True from migration 004 columns

.env.example

@@ -0,0 +1,57 @@
+IAAI_HEADLESS=true
+IAAI_LOG_LEVEL=INFO
+
+IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
+IAAI_MAX_CAPTURED_REQUESTS=40
+IAAI_MAX_CAPTURED_JSON_RESPONSES=20
+
+IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
+IAAI_LISTING_SEGMENTS=auto
+IAAI_MAX_PAGES_PER_RUN=999999
+IAAI_MAX_VEHICLES_PER_RUN=999999
+IAAI_PAGE_LINK_LIMIT=999999
+IAAI_INCLUDE_PAGINATION=true
+IAAI_COLLECT_CURRENT_PAGE_ONLY=false
+
+IAAI_HUMAN_PACE_ENABLED=true
+IAAI_PARALLEL_TABS=10
+CELERY_BATCH_SIZE=100
+IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
+IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
+IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
+IAAI_BETWEEN_VEHICLES_MIN_S=0.02
+IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
+
+IAAI_SYNC_ONLY_NEW=false
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
+
+IAAI_MAX_RETRIES=5
+IAAI_RETRY_DELAY_SECONDS=4
+IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
+IAAI_RETRY_JITTER_SECONDS=0.5
+IAAI_TIMEOUT_MS=90000
+IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
+
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+IAAI_DATABASE_ECHO=false
+IAAI_DATABASE_POOL_SIZE=5
+IAAI_DATABASE_MAX_OVERFLOW=5
+
+IAAI_REDIS_URL=redis://redis:6379/0
+
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CELERY_TASK_SOFT_TIME_LIMIT=86400
+CELERY_TASK_TIME_LIMIT=86520
+CELERY_BROKER_VISIBILITY_TIMEOUT=90000
+CELERY_WORKER_CONCURRENCY=1
+CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_BEAT_SYNC_LIMIT=0

.gitignore

@@ -3,8 +3,7 @@ __pycache__/
 .coverage
 coverage.xml
 .env*
-!.env
-.env.example
+!.env.example
 .venv/
 venv/
 *.pyc
@@ -21,3 +20,11 @@ artifacts/
 celerybeat-schedule*
 tokens_data/
 tokens.json
+
+# Local deployment/debug artifacts
+/*.tgz
+/*.tar.gz
+/*.zip
+/NOW()
+/_speed_check.sql
+/.env.bak*

Dockerfile

@@ -13,6 +13,10 @@ RUN pip install --no-cache-dir uv \
     && pip install --no-cache-dir -r /tmp/requirements.txt \
     && pip install --no-cache-dir playwright-stealth
 
+# Speed-up libs: orjson (fast JSON parse), brotli (HTTP br decompress).
+# Pinned outside uv.lock to avoid lock-regen churn.
+RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0"
+
 # Install both Chromium and Firefox. Chromium is the default engine in Docker
 # because it works more reliably with the current IAAI listing page.
 RUN python -m playwright install chromium firefox

README.md

@@ -112,7 +112,32 @@ docker compose up -d
 
 `entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд.
 
-По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
+По умолчанию `beat` запускает синхронизацию **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`).
+
+Текущая стратегия устойчива к багам пагинации IAAI:
+
+- **первый запуск** делает полный bootstrap через `sitemap`;
+- если bootstrap не завершился, автоматически продолжается с чекпоинта;
+- **после первого полного прохода** каждый час выполняется проход по `sitemap`.
+  По умолчанию включён режим **rolling refresh**:
+  - берутся все URL из sitemap,
+  - новые URL добавляются сразу,
+  - исчезнувшие URL помечаются как `is_sold=true`,
+  - уже записанные авто обновляются **батчами по кругу**, а не все разом.
+
+  Это даёт более стабильную нагрузку и снижает риск не уложиться в час.
+
+  Доступные режимы:
+  - `IAAI_HOURLY_MODE=rolling_refresh` — рекомендуемый продовый режим;
+  - `IAAI_HOURLY_MODE=full_refresh` — перепарсить все активные авто за один hourly цикл;
+  - `IAAI_HOURLY_MODE=diff` — только новые авто + sold.
+
+За счёт этого система:
+
+- не зависит от page 39 / 70 / 100,
+- не уходит в бесконечный цикл пагинации,
+- даёт стабильный hourly refresh уже записанных авто без монолитного полного hourly прохода,
+- остаётся быстрой и устойчивой при hourly sync.
 
 Swagger-документация: `http://localhost:8000/docs`
 

alembic/versions/004_add_ingestion_tables.py

@@ -0,0 +1,77 @@
+# Добавление таблиц для новой архитектуры ingestion: candidates, raw snapshots, parse results, retry queue
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "004_add_ingestion_tables"
+down_revision: Union[str, None] = "003_add_composite_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Таблица vehicle_candidates
+    op.create_table(
+        "vehicle_candidates",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("url", sa.String(), nullable=False, unique=True),
+        sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
+        sa.Column("priority", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
+    )
+
+    op.create_index("ix_vehicle_candidates_url", "vehicle_candidates", ["url"])
+    op.create_index("ix_vehicle_candidates_status_discovered", "vehicle_candidates", ["status", "discovered_at"])
+
+    # Таблица vehicle_raw_snapshots
+    op.create_table(
+        "vehicle_raw_snapshots",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        sa.Column("method", sa.String(20), nullable=False),
+        sa.Column("success", sa.Boolean(), nullable=False),
+        sa.Column("raw_data", sa.Text(), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+    )
+
+    op.create_index("ix_vehicle_raw_snapshots_candidate_id", "vehicle_raw_snapshots", ["candidate_id"])
+    op.create_index("ix_vehicle_raw_snapshots_captured_at", "vehicle_raw_snapshots", ["captured_at"])
+
+    # Таблица vehicle_parse_results
+    op.create_table(
+        "vehicle_parse_results",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("snapshot_id", sa.Integer(), sa.ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("parsed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        sa.Column("success", sa.Boolean(), nullable=False),
+        sa.Column("parsed_data", sa.Text(), nullable=True),
+        sa.Column("error_message", sa.Text(), nullable=True),
+    )
+
+    op.create_index("ix_vehicle_parse_results_snapshot_id", "vehicle_parse_results", ["snapshot_id"])
+    op.create_index("ix_vehicle_parse_results_parsed_at", "vehicle_parse_results", ["parsed_at"])
+
+    # Таблица vehicle_retry_queue
+    op.create_table(
+        "vehicle_retry_queue",
+        sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
+        sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("reason", sa.String(50), nullable=False),
+        sa.Column("retry_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
+    )
+
+    op.create_index("ix_vehicle_retry_queue_candidate_id", "vehicle_retry_queue", ["candidate_id"])
+    op.create_index("ix_vehicle_retry_queue_retry_at", "vehicle_retry_queue", ["retry_at"])
+
+
+def downgrade() -> None:
+    op.drop_table("vehicle_retry_queue")
+    op.drop_table("vehicle_parse_results")
+    op.drop_table("vehicle_raw_snapshots")
+    op.drop_table("vehicle_candidates")

deploy_vps.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+APP_DIR="/opt/iaai_scraper_project"
+REPO_URL="${1:-}"
+
+if [[ -z "${REPO_URL}" ]]; then
+  echo "Usage: ./deploy_vps.sh <git-repo-url>"
+  exit 1
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y --no-install-recommends \
+  ca-certificates \
+  curl \
+  git \
+  gnupg \
+  lsb-release
+
+install -m 0755 -d /etc/apt/keyrings
+if [[ ! -f /etc/apt/keyrings/docker.gpg ]]; then
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+fi
+
+chmod a+r /etc/apt/keyrings/docker.gpg
+ARCH="$(dpkg --print-architecture)"
+CODENAME="$(. /etc/os-release && echo "$VERSION_CODENAME")"
+echo \
+  "deb [arch=${ARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${CODENAME} stable" \
+  > /etc/apt/sources.list.d/docker.list
+
+apt-get update
+apt-get install -y --no-install-recommends \
+  docker-ce \
+  docker-ce-cli \
+  containerd.io \
+  docker-buildx-plugin \
+  docker-compose-plugin
+
+mkdir -p /opt
+
+if [[ -d "${APP_DIR}/.git" ]]; then
+  git -C "${APP_DIR}" fetch --all --prune
+  git -C "${APP_DIR}" reset --hard origin/HEAD
+else
+  rm -rf "${APP_DIR}"
+  git clone "${REPO_URL}" "${APP_DIR}"
+fi
+
+cd "${APP_DIR}"
+
+if [[ ! -f .env ]]; then
+  cp .env.vps.example .env
+fi
+
+docker compose down --remove-orphans || true
+docker compose up -d --build postgres redis migrate api worker beat
+docker compose ps

docker-compose.yml

@@ -5,16 +5,17 @@ x-app-env: &app-env
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
   CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
   IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300}
-  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600}
-  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
   # 0 => без лимита (в коде интерпретируется как None).
   # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
   CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
-  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
   CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
   IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
   IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
   IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
   IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
   IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
@@ -133,6 +134,8 @@ services:
     container_name: iaai-worker
     restart: unless-stopped
     init: true
+    mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
+    memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g}
     depends_on:
       migrate:
         condition: service_completed_successfully
@@ -141,9 +144,9 @@ services:
     stop_grace_period: 60s
     command: >
       celery -A iaai_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=prefork
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
       --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
     healthcheck:
       test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
       interval: 60s

iaai_scraper/browser/listing.py

@@ -3,13 +3,12 @@ import re
 import time
 from dataclasses import asdict, dataclass, field
 from typing import Any
-from urllib.parse import urljoin
+from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
 
 from playwright.sync_api import Page
 
 from .pace import HumanPacer
 from ..core.config import Settings
-from ..core.utils import first_non_empty
 
 logger = logging.getLogger("iaai_scraper.listing")
 VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
@@ -44,6 +43,7 @@ class ListingPageResult:
 
 
 class ListingCollector:
+	_DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE = 40
 	_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
 		"a[aria-label*='Next']",
 		"button[aria-label*='Next']",
@@ -103,7 +103,33 @@ class ListingCollector:
 			return None
 
 	@staticmethod
-	def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
+	def _get_vehicle_link_fingerprint(page: Page, limit: int = 5) -> tuple[str, ...]:
+		try:
+			values = page.evaluate(
+				"""
+				(limit) => {
+					return Array.from(document.querySelectorAll("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"))
+						.map((el) => (el.getAttribute('href') || '').trim())
+						.filter(Boolean)
+						.slice(0, limit);
+				}
+				""",
+				limit,
+			)
+			if not isinstance(values, list):
+				return tuple()
+			return tuple(str(value) for value in values if value)
+		except Exception:
+			return tuple()
+
+	@staticmethod
+	def _wait_for_navigation_result(
+		page: Page,
+		old_first_href: str,
+		expected_page_number: int | None,
+		*,
+		old_fingerprint: tuple[str, ...] = (),
+	) -> bool:
 		if old_first_href:
 			try:
 				page.wait_for_function(
@@ -117,9 +143,12 @@ class ListingCollector:
 			except Exception:
 				pass
 
+		new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
 		if expected_page_number is not None:
 			current_page = ListingCollector._get_current_page_number(page)
-			if current_page == expected_page_number:
+			if current_page == expected_page_number and (
+				not old_fingerprint or (new_fingerprint and new_fingerprint != old_fingerprint)
+			):
 				return True
 
 		try:
@@ -127,15 +156,85 @@ class ListingCollector:
 		except Exception:
 			pass
 
+		if not new_fingerprint:
+			new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
 		current_page = ListingCollector._get_current_page_number(page)
+		if new_fingerprint and old_fingerprint and new_fingerprint != old_fingerprint:
+			return current_page is None or expected_page_number is None or current_page == expected_page_number
 		if expected_page_number is not None and current_page == expected_page_number:
-			return True
+			return not old_fingerprint
 
-		return not old_first_href
+		return not old_first_href and (not old_fingerprint or bool(new_fingerprint))
 
-	def open_cars_listing(self, page: Page) -> None:
-		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
-		url = self.settings.listing.cars_url
+	def _extract_next_page_href(self, page: Page, expected_page_number: int | None = None) -> str | None:
+		try:
+			href = page.evaluate(
+				"""
+				(expectedPageNumber) => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const disabled = (el) => {
+						if (!el) return true;
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+					};
+
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
+						.filter((el) => visible(el) && !disabled(el));
+
+					const cleanHref = (el) => {
+						const href = (el?.getAttribute('href') || '').trim();
+						if (!href || href.startsWith('javascript:') || href.startsWith('#')) {
+							return '';
+						}
+						return href;
+					};
+
+					if (expectedPageNumber !== null && expectedPageNumber !== undefined) {
+						const numeric = controls.find((el) => {
+							const text = (el.textContent || '').trim();
+							return /^\d+$/.test(text) && parseInt(text, 10) === expectedPageNumber;
+						});
+						const numericHref = cleanHref(numeric);
+						if (numericHref) {
+							return numericHref;
+						}
+					}
+
+					const explicitNext = controls.find((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
+					});
+					return cleanHref(explicitNext);
+				}
+				""",
+				expected_page_number,
+			)
+			if not href:
+				return None
+			return urljoin(page.url or self.settings.home_url, str(href))
+		except Exception:
+			return None
+
+	@staticmethod
+	def _build_listing_page_url(url: str, page_number: int) -> str:
+		parsed = urlparse(url)
+		query_items = [
+			(key, value)
+			for key, value in parse_qsl(parsed.query, keep_blank_values=True)
+			if key.lower() not in {"page", "pagenumber", "currentpage"}
+		]
+		query_items.append(("page", str(page_number)))
+		return urlunparse(parsed._replace(query=urlencode(query_items)))
+
+	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
+		url = url_override or self.settings.listing.cars_url
+		logger.info("Opening cars listing page: %s", url)
 		last_err = None
 		for attempt in range(3):
 			try:
@@ -202,85 +301,132 @@ class ListingCollector:
 			# Короткая пауза вместо длинного sleep.
 			time.sleep(1.0)
 
-	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
-		applied = {"make": None, "model": None}
+	def _get_listing_html(self, page: Page, page_number: int) -> str:
+		try:
+			return page.locator("html").inner_html(timeout=5_000)
+		except Exception as exc:
+			logger.warning("listing html read failed on page %d: %s", page_number, exc)
+			return ""
+
+	def _has_next_page_from_html(self, html: str, current_page_number: int | None = None) -> bool:
+		if not html:
+			return False
+		normalized = html.replace("\\/", "/")
+		if re.search(
+			r"rel\s*=\s*['\"]next['\"]|aria-label\s*=\s*['\"][^'\"]*next|title\s*=\s*['\"][^'\"]*next|class\s*=\s*['\"][^'\"]*next|icon-arrow-right|arrow-right|>\s*next\s*<|>\s*[›»>]\s*<",
+			normalized,
+			re.IGNORECASE,
+		):
+			return True
+		if current_page_number is not None:
+			next_page = current_page_number + 1
+			if re.search(rf">\s*{next_page}\s*<", normalized, re.IGNORECASE):
+				return True
+			if re.search(rf"page={next_page}(?:\D|$)", normalized, re.IGNORECASE):
+				return True
+		return False
+
+	def apply_filters(
+		self,
+		page: Page,
+		make: str | None = None,
+		model: str | None = None,
+		year_min: int | None = None,
+		year_max: int | None = None,
+	) -> dict[str, str | int | None]:
+		applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
 		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
 			applied["make"] = make
 			self.pacer.after_filter_action()
 		if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
 			applied["model"] = model
 			self.pacer.after_filter_action()
+		if year_min is not None or year_max is not None:
+			if self._apply_year_range(page, year_min, year_max):
+				applied["year_min"] = year_min
+				applied["year_max"] = year_max
+				self.pacer.after_filter_action()
 		return applied
 
+	def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
+		"""Заполняет поля фильтра Year и нажимает Apply Year."""
+		if year_min is None and year_max is None:
+			return False
+		try:
+			success = page.evaluate(
+				"""([yearMin, yearMax]) => {
+					const inputs = Array.from(document.querySelectorAll('input'));
+					const yearInputs = inputs.filter(inp => {
+						const v = parseInt(inp.value, 10);
+						return !isNaN(v) && v >= 1900 && v <= 2100;
+					});
+					if (yearInputs.length < 2) return false;
+					yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
+					const setVal = (el, val) => {
+						const setter = Object.getOwnPropertyDescriptor(
+							HTMLInputElement.prototype, 'value'
+						).set;
+						setter.call(el, String(val));
+						el.dispatchEvent(new Event('input', {bubbles: true}));
+						el.dispatchEvent(new Event('change', {bubbles: true}));
+					};
+					if (yearMin !== null) setVal(yearInputs[0], yearMin);
+					if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
+					const container = yearInputs[0].closest(
+						'[class*="filter"], [class*="year"], section, fieldset'
+					) || yearInputs[0].parentElement.parentElement;
+					if (container) {
+						const btn = Array.from(container.querySelectorAll(
+							'button, a, [role="button"], span[class*="apply"]'
+						)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
+						if (btn) { btn.click(); return true; }
+					}
+					yearInputs[yearInputs.length - 1].dispatchEvent(
+						new KeyboardEvent('keydown', {
+							key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
+						})
+					);
+					return true;
+				}""",
+				[year_min, year_max],
+			)
+			if success:
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15_000)
+				except Exception:
+					pass
+				self._wait_for_listing_content(page)
+				logger.info("Applied year range filter: %s — %s", year_min, year_max)
+				return True
+		except Exception as exc:
+			logger.warning("Failed to apply year range filter: %s", exc)
+		return False
+
 	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
-		# Считываем ссылки одним проходом по DOM.
+		# Для IAAI HTML/hydration-извлечение стабильнее, чем прямой DOM eval.
 		self._accept_cookie_banner(page)
 		self._wait_for_listing_content(page)
-		try:
-			raw_items = page.eval_on_selector_all(
-				"a[href], [data-href], [href]",
-				"""
-				(nodes) => nodes.map((node) => ({
-					href:
-						node.getAttribute('href') ||
-						node.getAttribute('data-href') ||
-						node.getAttribute('data-url') ||
-						'',
-					title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
-					text: (node.textContent || '').trim(),
-				}))
-				""",
-			)
-		except Exception as exc:
-			logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
-			raw_items = []
-		total = min(len(raw_items), self.settings.listing.page_link_limit)
 		links: list[ListingVehicleLink] = []
 		seen: set[str] = set()
-		for idx in range(total):
-			item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
-			href = str(item.get("href") or "")
-			match = VEHICLE_HREF_RE.search(href)
-			if not match:
-				continue
-			lot_number = match.group(1)
-			absolute = urljoin(self.settings.home_url, match.group(0))
+		html = self._get_listing_html(page, page_number)
+		for absolute, lot_number in self._extract_vehicle_links_from_html(html):
 			if absolute in seen:
 				continue
 			seen.add(absolute)
-			title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
-			links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
+			links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
 			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
 
-		# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
-		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
-		if not links:
-			try:
-				page.wait_for_timeout(1_500)
-			except Exception:
-				pass
-			try:
-				html = page.content()
-			except Exception as exc:
-				logger.debug("page.content() failed on page %d: %s", page_number, exc)
-				html = ""
+		if links:
+			logger.info(
+				"Page %d: recovered %d vehicle links from HTML fallback",
+				page_number,
+				len(links),
+			)
 
-			for absolute, lot_number in self._extract_vehicle_links_from_html(html):
-				if absolute in seen:
-					continue
-				seen.add(absolute)
-				links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
-				if len(links) >= self.settings.listing.max_vehicles_per_run:
-					break
-
-			if links:
-				logger.info(
-					"Page %d: recovered %d vehicle links from HTML fallback",
-					page_number,
-					len(links),
-				)
-		next_page_detected = self._has_next_page(page)
+		next_page_detected = self._has_next_page_from_html(html, current_page_number=page_number)
+		if not next_page_detected and not html:
+			next_page_detected = self._has_next_page(page)
 		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
 
 	def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
@@ -307,12 +453,59 @@ class ListingCollector:
 	def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
 		# Запоминаем первую ссылку текущей страницы для определения смены контента.
 		old_first_href = ""
+		old_fingerprint: tuple[str, ...] = tuple()
 		try:
 			first_link = page.locator(VEHICLE_LINK_SELECTOR).first
 			if first_link.count() > 0:
 				old_first_href = first_link.get_attribute("href") or ""
 		except Exception:
 			pass
+		old_fingerprint = self._get_vehicle_link_fingerprint(page)
+
+		if expected_page_number is not None and expected_page_number > 1:
+			direct_page_url = self._build_listing_page_url(
+				page.url or self.settings.listing.cars_url,
+				expected_page_number,
+			)
+			try:
+				logger.debug("Navigating directly to listing page %d via URL: %s", expected_page_number, direct_page_url)
+				page.goto(direct_page_url, wait_until="domcontentloaded", timeout=15_000)
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+			except Exception as exc:
+				logger.debug("Direct page-number navigation failed for page %d via %s: %s", expected_page_number, direct_page_url, exc)
+
+		next_href = self._extract_next_page_href(page, expected_page_number)
+		if next_href:
+			try:
+				logger.debug("Navigating directly to next listing page: %s", next_href)
+				page.goto(next_href, wait_until="domcontentloaded", timeout=15_000)
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
+					self.pacer.after_page_change()
+					return True
+			except Exception as exc:
+				logger.debug("Direct next-page navigation failed for %s: %s", next_href, exc)
+
+		if (
+			expected_page_number is not None
+			and expected_page_number >= self._DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE
+		):
+			logger.warning(
+				"Deep pagination direct navigation failed for page %d; skipping flaky UI pagination fallbacks",
+				expected_page_number,
+			)
+			return False
 
 		for selector in self._NEXT_PAGE_SELECTORS:
 			locator = page.locator(selector).first
@@ -332,7 +525,12 @@ class ListingCollector:
 			except Exception:
 				continue
 
-			if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+			if self._wait_for_navigation_result(
+				page,
+				old_first_href,
+				expected_page_number,
+				old_fingerprint=old_fingerprint,
+			):
 				self.pacer.after_page_change()
 				return True
 
@@ -392,7 +590,12 @@ class ListingCollector:
 				"""
 			))
 			if clicked:
-				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
 					self.pacer.after_page_change()
 					return True
 		except Exception as exc:
@@ -423,7 +626,12 @@ class ListingCollector:
 				"""
 			))
 			if clicked:
-				if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
+				if self._wait_for_navigation_result(
+					page,
+					old_first_href,
+					expected_page_number,
+					old_fingerprint=old_fingerprint,
+				):
 					self.pacer.after_page_change()
 					return True
 		except Exception as exc:
@@ -508,10 +716,18 @@ class ListingCollector:
 				len(all_links) >= self.settings.listing.max_vehicles_per_run
 				or self.settings.listing.collect_current_page_only
 				or not self.settings.listing.include_pagination
-				or not page_result.next_page_detected
 			):
 				break
-			if not self.go_to_next_page(page):
+
+			blind_page_probe = not page_result.next_page_detected
+			if not self.go_to_next_page(page, expected_page_number=page_number + 1):
+				if blind_page_probe:
+					logger.info(
+						"Stopping pagination on page %d: direct page probe for %d failed and no next-page control was detected",
+						page_number,
+						page_number + 1,
+					)
+					break
 				break
 
 		return {
@@ -564,18 +780,45 @@ class ListingCollector:
 		try:
 			return bool(page.evaluate(
 				"""
-				() => Array.from(document.querySelectorAll('a,button,[role="button"]')).some((el) => {
-					const text = (el.textContent || '').trim().toLowerCase();
-					const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
-					const title = (el.getAttribute('title') || '').trim().toLowerCase();
-					const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
-					const cls = (el.getAttribute('class') || '').trim().toLowerCase();
-					const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
-					const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
-					const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
-					const looksNumericNext = /^\d+$/.test(text);
-					return !disabled && visible && (rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || looksNumericNext || ['next', '›', '»', '>'].includes(text));
-				})
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						return !disabled && visible(el);
+					});
+
+					const hasExplicitNext = controls.some((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (hasExplicitNext) {
+						return true;
+					}
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (!current) {
+						return false;
+					}
+
+					const currentPage = parseInt((current.textContent || '').trim(), 10);
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+					});
+				}
 				"""
 			))
 		except Exception:

iaai_scraper/cli.py

@@ -38,6 +38,17 @@ def build_parser() -> argparse.ArgumentParser:
     sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
     sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
 
+    # Новые команды для ingestion pipeline
+    subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap")
+
+    fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates")
+    fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process")
+
+    enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots")
+    enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process")
+
+    subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)")
+
     return parser
 
 
@@ -64,6 +75,39 @@ def main() -> None:
             data = scraper.scrape_vehicle_detail(args.vehicle_url)
         elif args.command == "sync-vehicle":
             data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
+        elif args.command == "sync-listing":
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                make=args.make,
+                model=args.model,
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+            )
+        elif args.command == "discover-vehicles":
+            from .discovery_service import DiscoveryService
+            discovery = DiscoveryService()
+            count = discovery.discover_new_vehicles()
+            print(f"Discovered {count} new vehicle candidates")
+            return
+        elif args.command == "fetch-pending":
+            from .fetch_service import FetchService
+            fetch = FetchService()
+            count = fetch.process_pending_candidates(limit=args.limit)
+            print(f"Successfully fetched {count} candidates")
+            return
+        elif args.command == "enrich-snapshots":
+            from .enrichment_service import EnrichmentService
+            enrichment = EnrichmentService()
+            count = enrichment.process_unparsed_snapshots(limit=args.limit)
+            print(f"Successfully enriched {count} snapshots")
+            return
+        elif args.command == "run-pipeline":
+            from .scheduler_service import SchedulerService
+            scheduler = SchedulerService()
+            scheduler.run_full_pipeline()
+            print("Pipeline completed")
+            return
         else:
             only_new = None if args.only_new is None else args.only_new == "true"
             data = scraper.sync_listing(

iaai_scraper/core/config.py

@@ -1,3 +1,4 @@
+import json
 import os
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -105,12 +106,102 @@ class ListingConfig:
     cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
     max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
     max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
+    resume_max_nav_pages: int = _env_int("IAAI_LISTING_RESUME_MAX_NAV_PAGES", 90)
     page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
     include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
     collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
     # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
     # прекращаем листать — все новые машины уже найдены. 0 = отключено.
     early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
+
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
+    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 3000)
+    sitemap_timeout_seconds: int = _env_int("IAAI_SITEMAP_TIMEOUT_SECONDS", 30)
+    sitemap_retry_attempts: int = _env_int("IAAI_SITEMAP_RETRY_ATTEMPTS", 4)
+    sitemap_retry_backoff_seconds: float = _env_float("IAAI_SITEMAP_RETRY_BACKOFF_SECONDS", 1.25)
+    sitemap_direct_probe_limit: int = _env_int("IAAI_SITEMAP_DIRECT_PROBE_LIMIT", 12)
+    sitemap_direct_probe_stop_after_misses: int = _env_int(
+        "IAAI_SITEMAP_DIRECT_PROBE_STOP_AFTER_MISSES",
+        3,
+    )
+    sitemap_use_curl_cffi: bool = _env_bool("IAAI_SITEMAP_USE_CURL_CFFI", True)
+
+
+# Список брендов IAAI для автоматической сегментации.
+# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
+IAAI_DEFAULT_MAKES: tuple[str, ...] = (
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
+    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
+    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
+    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
+    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
+    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
+    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
+    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
+    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
+    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
+)
+
+# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
+IAAI_PAGINATION_CEILING = 22_600
+
+# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
+_LARGE_MAKES: frozenset[str] = frozenset({
+    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
+    "KIA", "DODGE", "JEEP",
+})
+_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2019),
+    (2020, 2027),
+)
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
+    Крупные бренды автоматически разбиваются по диапазонам годов.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        segments: list[dict[str, str | int | None]] = []
+        for m in IAAI_DEFAULT_MAKES:
+            if m in _LARGE_MAKES:
+                for yr_min, yr_max in _YEAR_SPLITS:
+                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
+            else:
+                segments.append({"make": m, "year_min": None, "year_max": None})
+        return segments
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
 
 
 # - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
@@ -130,6 +221,9 @@ class DatabaseConfig:
 @dataclass(slots=True)
 class RedisConfig:
     url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
 
 
 # --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@@ -140,6 +234,7 @@ class CeleryConfig:
     result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
     task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
     task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
     worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
     worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
     broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
@@ -147,6 +242,7 @@ class CeleryConfig:
     beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
     batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
     parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
+    parallel_segments: bool = _env_bool("IAAI_PARALLEL_SEGMENTS", False)
     block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
 
 
@@ -201,6 +297,7 @@ class Settings:
     capture: CaptureConfig = field(default_factory=CaptureConfig)
     pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
     listing: ListingConfig = field(default_factory=ListingConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
     database: DatabaseConfig = field(default_factory=DatabaseConfig)
     redis: RedisConfig = field(default_factory=RedisConfig)
     celery: CeleryConfig = field(default_factory=CeleryConfig)

iaai_scraper/core/exceptions.py

@@ -8,3 +8,7 @@ class AntiBotDetectedError(ScraperError):
 
 class SiteStructureChangedError(ScraperError):
     """Вызывается, когда структура страницы изменилась и данных не хватает."""
+
+
+class ListingResumeError(ScraperError):
+    """Вызывается, когда resume по checkpoint больше недостижим."""

iaai_scraper/discovery/__init__.py

@@ -0,0 +1,17 @@
+from .sitemap import (
+    DEFAULT_SITEMAP_INDEX_URL,
+    SitemapDiscoveryError,
+    SitemapBlockedError,
+    SitemapDiscoveryResult,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+__all__ = [
+    "DEFAULT_SITEMAP_INDEX_URL",
+    "SitemapBlockedError",
+    "SitemapDiscoveryError",
+    "SitemapDiscoveryResult",
+    "discover_vehicle_urls_from_sitemap",
+    "discover_vehicle_urls_from_sitemap_with_stats",
+]

iaai_scraper/discovery/sitemap.py

@@ -0,0 +1,441 @@
+from __future__ import annotations
+
+import gzip
+import io
+import logging
+import random
+import re
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from typing import Iterable
+from urllib.error import HTTPError, URLError
+from urllib.parse import urljoin, urlsplit, urlunsplit
+from urllib.request import Request, urlopen
+
+from ..core.config import Settings
+
+try:
+    from curl_cffi import requests as curl_requests
+except ImportError:  # pragma: no cover - optional runtime dependency guard
+    curl_requests = None
+
+logger = logging.getLogger("iaai_scraper.discovery.sitemap")
+
+DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
+_XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL)
+_BLOCK_MARKERS = (
+    b"pardon our interruption",
+    b"incapsula",
+    b"please stand by",
+    b"_incapsula_resource",
+    b"as you were browsing something about your browser",
+)
+_HTML_MARKERS = (b"<html", b"<!doctype html", b"<script", b"document.getelementsbyclassname")
+_CURL_IMPERSONATE_CHOICES = ("chrome136", "chrome133", "chrome131", "safari184")
+
+
+class SitemapDiscoveryError(RuntimeError):
+    pass
+
+
+class SitemapBlockedError(SitemapDiscoveryError):
+    pass
+
+
+@dataclass(slots=True)
+class SitemapFetchResult:
+    url: str
+    payload: bytes
+    source: str
+    blocked: bool = False
+    status_code: int | None = None
+    content_type: str | None = None
+
+
+@dataclass(slots=True)
+class SitemapDiscoveryStats:
+    transport: str
+    fetched_sitemaps: int = 0
+    blocked_sitemaps: int = 0
+    malformed_sitemaps: int = 0
+    direct_probe_hits: int = 0
+    direct_probe_misses: int = 0
+
+
+@dataclass(slots=True)
+class SitemapDiscoveryResult:
+    vehicle_urls: list[str]
+    stats: SitemapDiscoveryStats
+
+
+_DEFAULT_HEADERS = {
+    "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Cache-Control": "no-cache",
+    "Pragma": "no-cache",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+class _SitemapDownloader:
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+        self.discovery = settings.discovery
+        self.proxy_url = settings.proxy.server
+        self.proxy_auth = None
+        if settings.proxy.username:
+            password = settings.proxy.password or ""
+            self.proxy_auth = (settings.proxy.username, password)
+        self._transport_name = self._resolve_transport_name()
+
+    @property
+    def transport_name(self) -> str:
+        return self._transport_name
+
+    def _resolve_transport_name(self) -> str:
+        if self.discovery.sitemap_use_curl_cffi and curl_requests is not None:
+            return "curl_cffi"
+        return "urllib"
+
+    def fetch(self, url: str) -> SitemapFetchResult:
+        attempts = max(1, int(self.discovery.sitemap_retry_attempts))
+        last_exc: Exception | None = None
+        for attempt in range(1, attempts + 1):
+            try:
+                result = self._fetch_once(url)
+                if result.blocked:
+                    raise SitemapBlockedError(f"Anti-bot page returned for {url}")
+                return result
+            except SitemapBlockedError as exc:
+                last_exc = exc
+                logger.warning(
+                    "Sitemap fetch blocked (attempt %d/%d, transport=%s): %s",
+                    attempt,
+                    attempts,
+                    self.transport_name,
+                    url,
+                )
+            except Exception as exc:
+                last_exc = exc
+                logger.warning(
+                    "Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s",
+                    attempt,
+                    attempts,
+                    self.transport_name,
+                    url,
+                    exc,
+                )
+            if attempt >= attempts:
+                break
+            time.sleep(self._backoff_delay(attempt))
+        if last_exc is None:
+            raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}")
+        if isinstance(last_exc, SitemapDiscoveryError):
+            raise last_exc
+        raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc
+
+    def _fetch_once(self, url: str) -> SitemapFetchResult:
+        if self.transport_name == "curl_cffi":
+            return self._fetch_with_curl_cffi(url)
+        return self._fetch_with_urllib(url)
+
+    def _build_headers(self) -> dict[str, str]:
+        headers = dict(_DEFAULT_HEADERS)
+        headers["User-Agent"] = self.settings.fingerprint.user_agent
+        return headers
+
+    def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult:
+        assert curl_requests is not None
+        response = curl_requests.get(
+            url,
+            headers=self._build_headers(),
+            timeout=self.discovery.sitemap_timeout_seconds,
+            impersonate=random.choice(_CURL_IMPERSONATE_CHOICES),
+            proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None,
+            proxy_auth=self.proxy_auth,
+            allow_redirects=True,
+        )
+        payload = self._decode_payload(
+            payload=response.content,
+            encoding=(response.headers.get("Content-Encoding") or ""),
+            url=url,
+        )
+        blocked = _looks_like_block_page(payload)
+        return SitemapFetchResult(
+            url=url,
+            payload=payload,
+            source="curl_cffi",
+            blocked=blocked,
+            status_code=int(response.status_code),
+            content_type=response.headers.get("Content-Type"),
+        )
+
+    def _fetch_with_urllib(self, url: str) -> SitemapFetchResult:
+        request = Request(url, headers=self._build_headers())
+        try:
+            with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response:
+                payload = response.read()
+                decoded = self._decode_payload(
+                    payload=payload,
+                    encoding=(response.headers.get("Content-Encoding") or ""),
+                    url=url,
+                )
+                return SitemapFetchResult(
+                    url=url,
+                    payload=decoded,
+                    source="urllib",
+                    blocked=_looks_like_block_page(decoded),
+                    status_code=getattr(response, "status", None),
+                    content_type=response.headers.get("Content-Type"),
+                )
+        except HTTPError as exc:
+            payload = exc.read() if hasattr(exc, "read") else b""
+            decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url)
+            blocked = exc.code in {403, 429} or _looks_like_block_page(decoded)
+            if blocked:
+                raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc
+            raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc
+        except URLError as exc:
+            raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc
+
+    @staticmethod
+    def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes:
+        normalized = (encoding or "").lower().strip()
+        if normalized == "gzip" or url.lower().endswith(".gz"):
+            try:
+                return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
+            except OSError:
+                return payload
+        return payload
+
+    def _backoff_delay(self, attempt: int) -> float:
+        base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds))
+        jitter = random.uniform(0.05, 0.35)
+        return base * (2 ** (attempt - 1)) + jitter
+
+
+def _is_vehicle_sitemap_url(url: str) -> bool:
+    lowered = url.strip().lower()
+    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
+        return False
+    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
+
+
+def _normalize_vehicle_url(url: str) -> str:
+    parts = urlsplit(url.strip())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+
+def _local_name(tag: str) -> str:
+    if "}" in tag:
+        return tag.rsplit("}", 1)[1]
+    return tag
+
+
+def _looks_like_block_page(payload: bytes) -> bool:
+    sample = payload[:8192].lower()
+    if any(marker in sample for marker in _BLOCK_MARKERS):
+        return True
+    if any(marker in sample for marker in _HTML_MARKERS) and b"<loc>" not in sample:
+        return True
+    return False
+
+
+def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
+    for match in _LOC_TAG_RE.finditer(xml_bytes):
+        try:
+            value = match.group(1).decode("utf-8", errors="ignore").strip()
+        except Exception:
+            continue
+        if value:
+            yield value
+
+
+def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
+    if _looks_like_block_page(xml_bytes):
+        raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML")
+    if not _XML_PREFIX_RE.search(xml_bytes[:256]):
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            yield from fallback_values
+            return
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as exc:
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            logger.warning(
+                "Falling back to regex sitemap loc extraction after XML parse error: %s",
+                exc,
+            )
+            yield from fallback_values
+            return
+        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
+
+    for element in root.iter():
+        if _local_name(element.tag) != "loc":
+            continue
+        if not element.text:
+            continue
+        value = element.text.strip()
+        if value:
+            yield value
+
+
+def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = _normalize_vehicle_url(url)
+        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
+            continue
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        result.append(normalized)
+    return result
+
+
+def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
+    return any("/vehicledetail/" in url.lower() for url in urls)
+
+
+def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]:
+    base_dir = index_url.rsplit("/", 1)[0] + "/"
+    return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)]
+
+
+def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]:
+    logger.info("Downloading sitemap index: %s", index_url)
+    try:
+        index_result = downloader.fetch(index_url)
+        sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)]
+        if sitemap_urls:
+            return sitemap_urls
+    except SitemapBlockedError as exc:
+        stats.blocked_sitemaps += 1
+        logger.warning("Sitemap index blocked, switching to direct probe: %s", exc)
+    except SitemapDiscoveryError as exc:
+        stats.malformed_sitemaps += 1
+        logger.warning("Sitemap index unusable, switching to direct probe: %s", exc)
+
+    logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe")
+    return _probe_direct_sitemap_urls(index_url, downloader, stats)
+
+
+def _probe_direct_sitemap_urls(
+    index_url: str,
+    downloader: _SitemapDownloader,
+    stats: SitemapDiscoveryStats,
+) -> list[str]:
+    discovered: list[str] = []
+    consecutive_misses = 0
+    probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit)
+
+    for sitemap_url in probe_urls:
+        try:
+            result = downloader.fetch(sitemap_url)
+            raw_urls = list(_iter_loc_values(result.payload))
+        except SitemapBlockedError:
+            stats.blocked_sitemaps += 1
+            consecutive_misses += 1
+            stats.direct_probe_misses += 1
+            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
+                break
+            continue
+        except SitemapDiscoveryError:
+            consecutive_misses += 1
+            stats.direct_probe_misses += 1
+            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
+                break
+            continue
+
+        if not raw_urls:
+            consecutive_misses += 1
+            stats.direct_probe_misses += 1
+            if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
+                break
+            continue
+
+        consecutive_misses = 0
+        stats.direct_probe_hits += 1
+        discovered.append(sitemap_url)
+
+    return discovered
+
+
+def _collect_vehicle_urls_from_sitemap(
+    sitemap_url: str,
+    downloader: _SitemapDownloader,
+    stats: SitemapDiscoveryStats,
+) -> list[str]:
+    logger.info("Downloading sitemap: %s", sitemap_url)
+    try:
+        result = downloader.fetch(sitemap_url)
+        raw_urls = list(_iter_loc_values(result.payload))
+    except SitemapBlockedError as exc:
+        stats.blocked_sitemaps += 1
+        logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc)
+        return []
+    except SitemapDiscoveryError as exc:
+        stats.malformed_sitemaps += 1
+        logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+        return []
+
+    stats.fetched_sitemaps += 1
+    vehicle_urls = _filter_vehicle_urls(raw_urls)
+    if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+        logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+        return []
+
+    logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+    return vehicle_urls
+
+
+def discover_vehicle_urls_from_sitemap(
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+    *,
+    settings: Settings | None = None,
+) -> list[str]:
+    result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings)
+    return result.vehicle_urls
+
+
+
+def discover_vehicle_urls_from_sitemap_with_stats(
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+    *,
+    settings: Settings | None = None,
+) -> SitemapDiscoveryResult:
+    runtime_settings = settings or Settings()
+    downloader = _SitemapDownloader(runtime_settings)
+    stats = SitemapDiscoveryStats(transport=downloader.transport_name)
+
+    sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats)
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats))
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError(
+            "No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue"
+        )
+
+    logger.info(
+        "Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)",
+        len(deduped),
+        stats.transport,
+        stats.fetched_sitemaps,
+        stats.blocked_sitemaps,
+        stats.malformed_sitemaps,
+        stats.direct_probe_hits,
+        stats.direct_probe_misses,
+    )
+    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

iaai_scraper/discovery_service.py

@@ -0,0 +1,129 @@
+import logging
+from datetime import datetime, timezone
+from typing import List
+
+from sqlalchemy.orm import Session
+
+from .core.config import settings
+from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError
+from .storage.db import get_db_session
+from .storage.models import VehicleCandidate
+
+logger = logging.getLogger("iaai_scraper.discovery_service")
+
+
+class DiscoveryService:
+    """Сервис для обнаружения новых автомобилей через sitemap IAAI."""
+
+    def __init__(self):
+        self.db: Session = get_db_session()
+
+    def discover_new_vehicles(self, max_urls: int = None) -> int:
+        """
+        Обнаруживает новые URL автомобилей и добавляет их в vehicle_candidates.
+
+        Args:
+            max_urls: Максимальное количество URL для обработки (None = все)
+
+        Returns:
+            Количество добавленных кандидатов
+        """
+        try:
+            logger.info("Starting vehicle discovery from sitemap")
+            result = discover_vehicle_urls_from_sitemap_with_stats()
+
+            if not result.urls:
+                logger.info("No new URLs discovered")
+                return 0
+
+            # Ограничиваем количество URL
+            urls_to_process = result.urls[:max_urls] if max_urls else result.urls
+            logger.info(f"Discovered {len(urls_to_process)} potential vehicle URLs (from {len(result.urls)} total)")
+
+            # Фильтруем уже существующие кандидаты
+            existing_urls = self._get_existing_candidate_urls(urls_to_process)
+            new_urls = [url for url in urls_to_process if url not in existing_urls]
+
+            if not new_urls:
+                logger.info("All discovered URLs already exist as candidates")
+                return 0
+
+            # Добавляем новых кандидатов
+            added_count = self._add_candidates(new_urls)
+            logger.info(f"Added {added_count} new vehicle candidates")
+
+            return added_count
+
+        except SitemapDiscoveryError as e:
+            logger.error(f"Sitemap discovery failed: {e}")
+            raise
+        except Exception as e:
+            logger.error(f"Unexpected error during discovery: {e}")
+            raise
+
+    def _get_existing_candidate_urls(self, urls: List[str]) -> set:
+        """Получает множество уже существующих URL кандидатов."""
+        if not urls:
+            return set()
+
+        # Разбиваем на батчи для эффективности
+        batch_size = 1000
+        existing = set()
+
+        for i in range(0, len(urls), batch_size):
+            batch = urls[i:i + batch_size]
+            result = self.db.query(VehicleCandidate.url).filter(
+                VehicleCandidate.url.in_(batch)
+            ).all()
+            existing.update(row[0] for row in result)
+
+        return existing
+
+    def _add_candidates(self, urls: List[str]) -> int:
+        """Добавляет новые кандидаты в базу данных."""
+        now = datetime.now(timezone.utc)
+        candidates = [
+            VehicleCandidate(
+                url=url,
+                discovered_at=now,
+                status="pending",
+                priority=0
+            )
+            for url in urls
+        ]
+
+        self.db.add_all(candidates)
+        self.db.commit()
+
+        return len(candidates)
+
+    def get_pending_candidates(self, limit: int = 100) -> List[VehicleCandidate]:
+        """Получает кандидатов в статусе 'pending' для обработки."""
+        return self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.status == "pending"
+        ).order_by(VehicleCandidate.priority.desc(), VehicleCandidate.discovered_at).limit(limit).all()
+
+    def mark_candidate_processing(self, candidate_id: int):
+        """Помечает кандидата как обрабатываемого."""
+        self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.id == candidate_id
+        ).update({
+            "status": "processing",
+            "last_attempt_at": datetime.now(timezone.utc),
+            "attempts": VehicleCandidate.attempts + 1
+        })
+        self.db.commit()
+
+    def mark_candidate_processed(self, candidate_id: int):
+        """Помечает кандидата как обработанного."""
+        self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.id == candidate_id
+        ).update({"status": "processed"})
+        self.db.commit()
+
+    def mark_candidate_failed(self, candidate_id: int):
+        """Помечает кандидата как неудачного."""
+        self.db.query(VehicleCandidate).filter(
+            VehicleCandidate.id == candidate_id
+        ).update({"status": "failed"})
+        self.db.commit()

iaai_scraper/enrichment_service.py

@@ -0,0 +1,140 @@
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+from .core.config import settings
+from .parsing.mapper import CarMapper
+from .parsing.parser import VehicleParser
+from .storage.db import get_db_session
+from .storage.models import VehicleRawSnapshot, VehicleParseResult, Car
+from .storage.schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.enrichment_service")
+
+
+class EnrichmentService:
+    """Сервис для парсинга сырых данных и обогащения автомобилей."""
+
+    def __init__(self):
+        self.db: Session = get_db_session()
+        self.parser = VehicleParser()
+        self.mapper = CarMapper()
+
+    def enrich_vehicle(self, snapshot: VehicleRawSnapshot) -> Optional[VehicleParseResult]:
+        """
+        Парсит snapshot и сохраняет результат.
+
+        Args:
+            snapshot: Raw snapshot для парсинга
+
+        Returns:
+            VehicleParseResult если парсинг успешен
+        """
+        logger.info(f"Enriching snapshot {snapshot.id} for candidate {snapshot.candidate_id}")
+
+        if not snapshot.success or not snapshot.raw_data:
+            logger.warning(f"Snapshot {snapshot.id} has no data to parse")
+            return self._save_parse_result(snapshot.id, False, None, "No raw data available")
+
+        try:
+            # Парсим данные
+            parsed_data = self._parse_raw_data(snapshot.raw_data)
+            if not parsed_data:
+                return self._save_parse_result(snapshot.id, False, None, "Parsing failed")
+
+            # Маппим в CarRecord
+            car_record = self._map_to_car_record(parsed_data)
+            if not car_record:
+                return self._save_parse_result(snapshot.id, False, None, "Mapping failed")
+
+            # Сохраняем в cars таблицу
+            self._save_car(car_record)
+
+            # Сохраняем успешный результат парсинга
+            return self._save_parse_result(snapshot.id, True, json.dumps(parsed_data))
+
+        except Exception as e:
+            error_msg = f"Unexpected error during enrichment: {str(e)}"
+            logger.error(f"Error enriching snapshot {snapshot.id}: {error_msg}")
+            return self._save_parse_result(snapshot.id, False, None, error_msg)
+
+    def _parse_raw_data(self, raw_data: str) -> Optional[dict]:
+        """Парсит сырые данные в словарь."""
+        try:
+            # Если это JSON от browser capture
+            if raw_data.strip().startswith('{'):
+                data = json.loads(raw_data)
+                # Извлекаем vehicle_summary из scrape result
+                return data.get("vehicle_summary", {})
+
+            # Если HTML, используем VehicleParser
+            parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
+            return parsed.get("vehicle_summary", {})
+
+        except json.JSONDecodeError:
+            # Если не JSON, пробуем как HTML
+            try:
+                parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
+                return parsed.get("vehicle_summary", {})
+            except Exception:
+                return None
+
+    def _map_to_car_record(self, parsed_data: dict) -> Optional[CarRecord]:
+        """Маппит parsed data в CarRecord."""
+        try:
+            # Используем CarMapper
+            payload_insights = {}  # TODO: extract from raw data if needed
+            return self.mapper.map_to_car_record("dummy_url", parsed_data, payload_insights)
+        except Exception as e:
+            logger.error(f"Mapping failed: {e}")
+            return None
+
+    def _save_car(self, car_record: CarRecord):
+        """Сохраняет CarRecord в базу данных."""
+        # Используем PersistenceService
+        from .storage.db import PersistenceService
+        from .core.config import settings
+        persistence = PersistenceService(settings)
+        persistence.create_tables()
+        upsert_result = persistence.upsert_car(car_record)
+        logger.info(f"Car upserted: {upsert_result}")
+
+    def _save_parse_result(self, snapshot_id: int, success: bool,
+                          parsed_data: Optional[str], error_message: Optional[str] = None) -> VehicleParseResult:
+        """Сохраняет результат парсинга."""
+        result = VehicleParseResult(
+            snapshot_id=snapshot_id,
+            success=success,
+            parsed_data=parsed_data,
+            error_message=error_message
+        )
+        self.db.add(result)
+        self.db.commit()
+        self.db.refresh(result)
+        return result
+
+    def process_unparsed_snapshots(self, limit: int = 10) -> int:
+        """
+        Обрабатывает snapshots без результатов парсинга.
+
+        Returns:
+            Количество успешно обогащенных snapshots
+        """
+        # Находим snapshots без parse results
+        snapshots = self.db.query(VehicleRawSnapshot).outerjoin(
+            VehicleParseResult, VehicleRawSnapshot.id == VehicleParseResult.snapshot_id
+        ).filter(
+            VehicleRawSnapshot.success == True,
+            VehicleParseResult.id.is_(None)
+        ).limit(limit).all()
+
+        enriched_count = 0
+        for snapshot in snapshots:
+            result = self.enrich_vehicle(snapshot)
+            if result and result.success:
+                enriched_count += 1
+
+        return enriched_count

iaai_scraper/fetch_service.py

@@ -0,0 +1,119 @@
+import logging
+from datetime import datetime, timezone
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+from .core.config import settings
+from .scraper import IAAIScraper
+from .storage.db import get_db_session
+from .storage.models import VehicleCandidate, VehicleRawSnapshot
+
+logger = logging.getLogger("iaai_scraper.fetch_service")
+
+
+class FetchService:
+    """Сервис для захвата сырых данных автомобилей (HTTP/browser fallback)."""
+
+    def __init__(self):
+        self.db: Session = get_db_session()
+        self.scraper = IAAIScraper()
+
+    def fetch_vehicle_data(self, candidate: VehicleCandidate) -> Optional[VehicleRawSnapshot]:
+        """
+        Захватывает данные для кандидата автомобиля.
+
+        Args:
+            candidate: Кандидат для обработки
+
+        Returns:
+            VehicleRawSnapshot если захват успешен, None если неудача
+        """
+        logger.info(f"Fetching data for candidate {candidate.id}: {candidate.url}")
+
+        try:
+            # Сначала пытаемся HTTP fast-path
+            raw_data = self._try_http_capture(candidate.url)
+            if raw_data:
+                return self._save_snapshot(candidate.id, "http", True, raw_data)
+
+            # Если HTTP не сработал, используем browser fallback
+            logger.info(f"HTTP failed for {candidate.url}, trying browser fallback")
+            raw_data = self._try_browser_capture(candidate.url)
+            if raw_data:
+                return self._save_snapshot(candidate.id, "browser", True, raw_data)
+
+            # Оба метода failed
+            logger.warning(f"Both HTTP and browser capture failed for {candidate.url}")
+            self._save_snapshot(candidate.id, "browser", False, None, "Both capture methods failed")
+            return None
+
+        except Exception as e:
+            error_msg = f"Unexpected error during capture: {str(e)}"
+            logger.error(f"Error fetching {candidate.url}: {error_msg}")
+            self._save_snapshot(candidate.id, "unknown", False, None, error_msg)
+            return None
+
+    def _try_http_capture(self, url: str) -> Optional[str]:
+        """Пытается захватить данные через HTTP."""
+        try:
+            # Используем существующий метод из scraper
+            # Но нам нужно инициализировать scraper с сессией
+            # Пока заглушка - интегрируем позже
+            # Для теста вернем None, чтобы использовать browser
+            return None
+        except Exception as e:
+            logger.debug(f"HTTP capture failed for {url}: {e}")
+            return None
+
+    def _try_browser_capture(self, url: str) -> Optional[str]:
+        """Пытается захватить данные через browser."""
+        try:
+            # Используем scrape_vehicle_detail из scraper
+            with IAAIScraper() as scraper:
+                result = scraper.scrape_vehicle_detail(url)
+                # Возвращаем HTML или JSON данные
+                return result.get("raw_html") or json.dumps(result)
+        except Exception as e:
+            logger.debug(f"Browser capture failed for {url}: {e}")
+            return None
+
+    def _save_snapshot(self, candidate_id: int, method: str, success: bool,
+                      raw_data: Optional[str], error_message: Optional[str] = None) -> VehicleRawSnapshot:
+        """Сохраняет snapshot в базу данных."""
+        snapshot = VehicleRawSnapshot(
+            candidate_id=candidate_id,
+            method=method,
+            success=success,
+            raw_data=raw_data,
+            error_message=error_message
+        )
+        self.db.add(snapshot)
+        self.db.commit()
+        self.db.refresh(snapshot)
+        return snapshot
+
+    def process_pending_candidates(self, limit: int = 10) -> int:
+        """
+        Обрабатывает ожидающих кандидатов.
+
+        Returns:
+            Количество успешно обработанных кандидатов
+        """
+        from .discovery_service import DiscoveryService
+        discovery = DiscoveryService()
+
+        candidates = discovery.get_pending_candidates(limit)
+        processed_count = 0
+
+        for candidate in candidates:
+            discovery.mark_candidate_processing(candidate.id)
+
+            snapshot = self.fetch_vehicle_data(candidate)
+            if snapshot and snapshot.success:
+                discovery.mark_candidate_processed(candidate.id)
+                processed_count += 1
+            else:
+                discovery.mark_candidate_failed(candidate.id)
+
+        return processed_count

iaai_scraper/parsing/mapper.py

@@ -379,7 +379,7 @@ class CarMapper:
 		digest = hashlib.sha256(origin_id.encode()).digest()
 		alphabet = cls._PARSER_ID_ALPHABET
 		base = len(alphabet)
-		num = int.from_bytes(digest[:17], "big")  # 17 bytes = 136 bits, enough for 22 chars
+		num = int.from_bytes(digest[:17], "big")  # 17 байт = 136 бит, хватает на 22 символа
 		chars: list[str] = []
 		for _ in range(22):
 			num, idx = divmod(num, base)

iaai_scraper/scheduler_service.py

@@ -0,0 +1,69 @@
+import logging
+import time
+from datetime import datetime, timezone, timedelta
+
+from .core.config import settings
+from .discovery_service import DiscoveryService
+from .fetch_service import FetchService
+from .enrichment_service import EnrichmentService
+
+logger = logging.getLogger("iaai_scraper.scheduler_service")
+
+
+class SchedulerService:
+    """Сервис для планирования и координации ingestion pipeline."""
+
+    def __init__(self):
+        self.discovery = DiscoveryService()
+        self.fetch = FetchService()
+        self.enrichment = EnrichmentService()
+
+    def run_full_pipeline(self):
+        """Запускает полный цикл ingestion: discovery -> fetch -> enrichment."""
+        logger.info("Starting full ingestion pipeline")
+
+        try:
+            # 1. Discovery phase
+            logger.info("Phase 1: Discovery")
+            discovered_count = self.discovery.discover_new_vehicles()
+            logger.info(f"Discovered {discovered_count} new candidates")
+
+            # 2. Fetch phase
+            logger.info("Phase 2: Fetch")
+            fetched_count = self.fetch.process_pending_candidates(limit=50)
+            logger.info(f"Successfully fetched {fetched_count} candidates")
+
+            # 3. Enrichment phase
+            logger.info("Phase 3: Enrichment")
+            enriched_count = self.enrichment.process_unparsed_snapshots(limit=50)
+            logger.info(f"Successfully enriched {enriched_count} snapshots")
+
+            logger.info("Ingestion pipeline completed")
+
+        except Exception as e:
+            logger.error(f"Pipeline failed: {e}")
+            raise
+
+    def run_continuous_pipeline(self, interval_minutes: int = 30):
+        """Запускает непрерывный цикл ingestion с интервалом."""
+        logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals")
+
+        while True:
+            try:
+                self.run_full_pipeline()
+            except Exception as e:
+                logger.error(f"Pipeline iteration failed: {e}")
+
+            logger.info(f"Sleeping for {interval_minutes} minutes")
+            time.sleep(interval_minutes * 60)
+
+    def run_targeted_enrichment(self):
+        """Запускает только enrichment для существующих snapshots."""
+        logger.info("Running targeted enrichment")
+        enriched_count = self.enrichment.process_unparsed_snapshots(limit=100)
+        logger.info(f"Enriched {enriched_count} snapshots")
+
+    def cleanup_old_data(self, days_to_keep: int = 30):
+        """Очищает старые данные (опционально)."""
+        # TODO: implement if needed
+        pass

iaai_scraper/storage/db.py

@@ -14,6 +14,13 @@ from .schemas import CarRecord
 logger = logging.getLogger("iaai_scraper.db")
 
 
+def get_db_session() -> Session:
+    """Получить новую сессию базы данных."""
+    settings = Settings()
+    persistence = PersistenceService(settings)
+    return persistence.session_factory()
+
+
 CAR_DB_FIELDS = {
 	col.key for col in Car.__table__.columns
 	if col.key not in ("id",)
@@ -517,21 +524,38 @@ class PersistenceService:
 		is_postgres = "postgresql" in self.settings.database.url
 
 		with self.session_scope() as session:
+			# Страховка от ложного mark_sold при битом/неполном full-scan:
+			# если текущий список активных URL аномально мал относительно уже активных машин в БД,
+			# ничего не помечаем проданным.
+			active_db_count = int(session.execute(
+				select(func.count())
+				.select_from(Car)
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like(f"{lane}:%"))
+			).scalar_one() or 0)
+
+			current_active_count = len(active_origin_urls)
+			if active_db_count >= 1000 and current_active_count < max(500, int(active_db_count * 0.25)):
+				logger.warning(
+					"Skipping mark_sold: suspiciously small active set (%d URLs vs %d active in DB)",
+					current_active_count,
+					active_db_count,
+				)
+				return 0
+
 			if is_postgres:
 				# Создаём временную таблицу с активными URL.
-				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT PRIMARY KEY) ON COMMIT DROP"))
 				session.execute(text("TRUNCATE _active_urls"))
 
-				# Вставляем активные URL чанками.
+				# Вставляем активные URL чанками через executemany.
 				url_list = list(active_origin_urls)
 				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
 					chunk = url_list[i:i + _IN_CHUNK_SIZE]
-					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
-					params = {f"u{j}": url for j, url in enumerate(chunk)}
-					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
-
-				# Создаём индекс на временной таблице для ускорения JOIN.
-				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
+					session.execute(
+						text("INSERT INTO _active_urls (url) VALUES (:url) ON CONFLICT DO NOTHING"),
+						[{"url": url} for url in chunk],
+					)
 
 				# Массовая пометка проданных в PostgreSQL.
 				result = session.execute(text("""
@@ -543,10 +567,10 @@ class PersistenceService:
 						LEFT JOIN _active_urls a ON c.origin_url = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
-						AND c.origin_id LIKE 'iaai:%%'
+						AND c.origin_id LIKE :lane_prefix
 					) sub
 					WHERE cars.id = sub.id
-				"""))
+				"""), {"lane_prefix": f"{lane}:%"})
 				count = result.rowcount or 0
 			else:
 				# Упрощённый путь для SQLite.
@@ -554,7 +578,7 @@ class PersistenceService:
 					update(Car)
 					.where(Car.origin_url.notin_(active_origin_urls))
 					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like("iaai:%"))
+					.where(Car.origin_id.like(f"{lane}:%"))
 					.values(is_sold=True)
 				)
 				result = session.execute(stmt)
@@ -574,3 +598,42 @@ class PersistenceService:
 				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
 			)
 			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все активные (не sold) origin_url для указанного lane/prefix."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		*,
+		prefix: str = "iaai:",
+		offset: int = 0,
+		limit: int = 3000,
+	) -> list[str]:
+		"""Возвращает батч активных origin_url для циклического hourly refresh."""
+		with self.session_scope() as session:
+			rows = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.order_by(Car.last_seen_at.asc(), Car.id.asc())
+				.offset(max(0, int(offset)))
+				.limit(max(1, int(limit)))
+			).all()
+			return [str(row[0]) for row in rows if row and row[0]]
+
+	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		with self.session_scope() as session:
+			return int(session.execute(
+				select(func.count())
+				.select_from(Car)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+			).scalar_one() or 0)

iaai_scraper/storage/models.py

@@ -80,3 +80,65 @@ class SyncRun(Base):
 	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class VehicleCandidate(Base):
+	__tablename__ = "vehicle_candidates"
+	__table_args__ = (
+		Index("ix_vehicle_candidates_url", "url"),
+		Index("ix_vehicle_candidates_status_discovered", "status", "discovered_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	url: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
+	discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True)  # pending, processing, processed, failed
+	priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	raw_snapshots: Mapped[list["VehicleRawSnapshot"]] = relationship("VehicleRawSnapshot", back_populates="candidate", cascade="all, delete-orphan")
+
+
+class VehicleRawSnapshot(Base):
+	__tablename__ = "vehicle_raw_snapshots"
+	__table_args__ = (
+		Index("ix_vehicle_raw_snapshots_candidate_id", "candidate_id"),
+		Index("ix_vehicle_raw_snapshots_captured_at", "captured_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
+	candidate: Mapped[VehicleCandidate] = relationship("VehicleCandidate", back_populates="raw_snapshots")
+	captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	method: Mapped[str] = mapped_column(String(20), nullable=False)  # http, browser
+	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
+	raw_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # HTML or JSON content
+	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+	parse_results: Mapped[list["VehicleParseResult"]] = relationship("VehicleParseResult", back_populates="snapshot", cascade="all, delete-orphan")
+
+
+class VehicleParseResult(Base):
+	__tablename__ = "vehicle_parse_results"
+	__table_args__ = (
+		Index("ix_vehicle_parse_results_snapshot_id", "snapshot_id"),
+		Index("ix_vehicle_parse_results_parsed_at", "parsed_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	snapshot_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True)
+	snapshot: Mapped[VehicleRawSnapshot] = relationship("VehicleRawSnapshot", back_populates="parse_results")
+	parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
+	parsed_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # JSON with parsed vehicle data
+	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class VehicleRetryQueue(Base):
+	__tablename__ = "vehicle_retry_queue"
+	__table_args__ = (
+		Index("ix_vehicle_retry_queue_candidate_id", "candidate_id"),
+		Index("ix_vehicle_retry_queue_retry_at", "retry_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
+	reason: Mapped[str] = mapped_column(String(50), nullable=False)  # parse_failed, capture_failed, etc.
+	retry_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
+	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)

iaai_scraper/worker/celery_app.py

@@ -4,11 +4,13 @@ import logging
 
 from celery import Celery
 from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
+from redis import Redis
 
 from ..core.config import settings
 from ..core.logs import setup_logging
 
 logger = logging.getLogger("iaai_scraper.worker.celery_app")
+STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
 
 
 @celery_setup_logging.connect
@@ -96,6 +98,24 @@ celery_app.autodiscover_tasks(["iaai_scraper.worker"])
 def _on_worker_ready(**kwargs):
     """Сразу при старте worker отправляем первую задачу sync_listing,
     чтобы не ждать час до первого beat-цикла."""
+    try:
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+    except Exception:
+        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        return
+
+    if not should_dispatch:
+        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
+        return
+
     logger.info("Worker ready — dispatching initial sync_listing task")
     celery_app.send_task(
         "iaai_scraper.worker.tasks.sync_listing_task",

pyproject.toml

@@ -11,6 +11,7 @@ requires-python = ">=3.11"
 dependencies = [
   "alembic>=1.14.0",
   "celery>=5.4.0",
+  "curl-cffi>=0.11.0",
   "fastapi>=0.115.0",
   "playwright>=1.53.0",
   "psycopg2-binary>=2.9.9",
@@ -20,6 +21,8 @@ dependencies = [
   "sqlalchemy>=2.0.32",
   "uvicorn>=0.34.0",
   "urllib3>=2.0.0",
+  "orjson>=3.10.0",
+  "brotli>=1.1.0",
 ]
 
 [project.optional-dependencies]

tests/test_listing.py

@@ -8,45 +8,98 @@ from iaai_scraper.browser.listing import ListingCollector
 
 
 class _FakePage:
-    def __init__(self, counts: dict[str, int]) -> None:
+    def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None:
         self._counts = counts
+        self._attrs = attrs or {}
         self._evaluate_result = False
         self._evaluate_values: list[object] = []
+        self._html = ""
+        self.url = "https://www.iaai.com/Vehiclelisting/Cars"
+        self._current_page_number: int | None = None
+        self._vehicle_hrefs: list[str] = []
+        self._wait_for_function_error: Exception | None = None
+        self._clicks: dict[str, int] = {}
+        self._goto_calls: list[str] = []
 
     class _Locator:
-        def __init__(self, count_value: int) -> None:
+        def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None:
+            self._page = page
+            self._selector = selector
             self._count_value = count_value
 
+        @property
+        def first(self) -> "_FakePage._Locator":
+            return self
+
         def count(self) -> int:
             return self._count_value
 
-    def locator(self, selector: str) -> "_FakePage._Locator":
-        return _FakePage._Locator(self._counts.get(selector, 0))
+        def is_visible(self, timeout: int | None = None) -> bool:
+            _ = timeout
+            return False
 
-    def evaluate(self, _script: str):
+        def get_attribute(self, name: str, timeout: int | None = None) -> str | None:
+            _ = timeout
+            return self._page._attrs.get(self._selector, {}).get(name)
+
+        def click(self, timeout: int | None = None) -> None:
+            _ = timeout
+            self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1
+            return None
+
+    class _HtmlLocator:
+        def __init__(self, html: str) -> None:
+            self._html = html
+
+        def inner_html(self, timeout: int | None = None) -> str:
+            _ = timeout
+            return self._html
+
+    def locator(self, selector: str):
+        if selector == "html":
+            return _FakePage._HtmlLocator(self._html)
+        return _FakePage._Locator(self, selector, self._counts.get(selector, 0))
+
+    def evaluate(self, script: str, *args):
+        _ = args
         if self._evaluate_values:
             return self._evaluate_values.pop(0)
+        if "querySelectorAll" in script and "VehicleDetail" in script:
+            return list(self._vehicle_hrefs)
+        if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script:
+            return self._current_page_number if self._current_page_number is not None else self._evaluate_result
         return self._evaluate_result
 
+    def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
+        _ = selector, timeout
+        return None
+
+    def wait_for_function(self, script: str, timeout: int | None = None) -> None:
+        _ = script, timeout
+        if self._wait_for_function_error is not None:
+            raise self._wait_for_function_error
+        return None
+
+    def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None:
+        _ = wait_until, timeout
+        self._goto_calls.append(url)
+        self.url = url
+
 
 class TestListingUnit(unittest.TestCase):
-    def test_has_next_page_true_for_known_selector(self) -> None:
-        page = _FakePage({"a[aria-label*='Next']": 1})
-        self.assertTrue(ListingCollector._has_next_page(page))
-
-    def test_has_next_page_false_when_no_selectors(self) -> None:
-        page = _FakePage({})
-        self.assertFalse(ListingCollector._has_next_page(page))
-
-    def test_has_next_page_true_for_numeric_pagination_fallback(self) -> None:
+    def test_pagination_detection_and_page_number(self) -> None:
+        # Есть кнопка Next → True.
+        self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']":  1})))
+        # Нет селекторов → False.
+        self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
+        # Числовая пагинация через JS → True только если evaluate явно нашёл next.
         page = _FakePage({})
         page._evaluate_result = True
         self.assertTrue(ListingCollector._has_next_page(page))
-
-    def test_get_current_page_number_from_text_counter(self) -> None:
-        page = _FakePage({})
-        page._evaluate_values = [5]
-        self.assertEqual(ListingCollector._get_current_page_number(page), 5)
+        # Номер текущей страницы.
+        page2 = _FakePage({})
+        page2._evaluate_values = [5]
+        self.assertEqual(ListingCollector._get_current_page_number(page2), 5)
 
     def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
         collector = ListingCollector(Settings(), HumanPacer(Settings()))
@@ -67,6 +120,67 @@ class TestListingUnit(unittest.TestCase):
             ],
         )
 
+    def test_has_next_page_from_html(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        html = '<a class="pagination-next" href="/Vehiclelisting/Cars?page=43">Next</a>'
+        self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42))
+        self.assertFalse(collector._has_next_page_from_html("<div>done</div>", current_page_number=42))
+
+    def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+
+        url = collector._build_listing_page_url(
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar",
+            44,
+        )
+
+        self.assertEqual(
+            url,
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44",
+        )
+
+    def test_collect_current_page_uses_html_first(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        page = _FakePage({})
+        page._html = '<a href="/VehicleDetail/555~US">Car 555</a><a class="pagination-next" href="/Vehiclelisting/Cars?page=2">Next</a>'
+
+        result = collector.collect_current_page(page, page_number=1)
+
+        self.assertEqual(len(result.vehicle_links), 1)
+        self.assertEqual(result.vehicle_links[0].lot_number, "555")
+        self.assertTrue(result.next_page_detected)
+
+    def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None:
+        page = _FakePage({})
+        page._wait_for_function_error = RuntimeError("same content")
+        page._current_page_number = 41
+        page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
+
+        self.assertFalse(
+            ListingCollector._wait_for_navigation_result(
+                page,
+                "/VehicleDetail/111~US",
+                41,
+                old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"),
+            )
+        )
+
+    def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        page = _FakePage(
+            {ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1},
+            attrs={
+                "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"},
+            },
+        )
+        page._wait_for_function_error = RuntimeError("same content")
+        page._current_page_number = 41
+        page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
+
+        self.assertFalse(collector.go_to_next_page(page, expected_page_number=41))
+        self.assertGreaterEqual(len(page._goto_calls), 1)
+        self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_mappers.py

@@ -41,30 +41,27 @@ class TestCarMapper(unittest.TestCase):
 
         self.assertFalse(record.is_damaged)
 
-    def test_unknown_and_empty_values_fallbacks(self) -> None:
+    def test_unknown_empty_values_and_normalization(self) -> None:
         record = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
             vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
-
         self.assertEqual(record.brand, "UNKNOWN")
         self.assertEqual(record.model, "UNKNOWN")
-        self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
         self.assertEqual(record.drive, "NA")
         self.assertEqual(record.gearbox, "NA")
 
-    def test_mapper_handles_case_and_spaces(self) -> None:
-        record = self.mapper.map_to_car_record(
+        # Нормализация регистра и пробелов.
+        record2 = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
+        self.assertEqual(record2.drive, "FWD")
+        self.assertEqual(record2.gearbox, "AT")
 
-        self.assertEqual(record.drive, "FWD")
-        self.assertEqual(record.gearbox, "AT")
-
-    def test_price_parsing_dirty_formats(self) -> None:
+    def test_price_and_currency_parsing(self) -> None:
         record = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
@@ -76,11 +73,9 @@ class TestCarMapper(unittest.TestCase):
                 "images": {},
             },
         )
-
         self.assertEqual(record.price, 5200)
 
-    def test_currency_detection_from_symbol(self) -> None:
-        record = self.mapper.map_to_car_record(
+        record2 = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
             payload_insights={
@@ -91,9 +86,8 @@ class TestCarMapper(unittest.TestCase):
                 "images": {},
             },
         )
-
-        self.assertEqual(record.currency, "EUR")
-        self.assertEqual(record.price, 4500)
+        self.assertEqual(record2.currency, "EUR")
+        self.assertEqual(record2.price, 4500)
 
 
 if __name__ == "__main__":

tests/test_parser.py

@@ -9,7 +9,7 @@ class TestVehicleParserUnit(unittest.TestCase):
     def setUp(self) -> None:
         self.parser = VehicleParser()
 
-    def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
+    def test_parse_dom_pairs_and_title(self) -> None:
         dom_text = """
         Stock #:
         45089484
@@ -23,41 +23,27 @@ class TestVehicleParserUnit(unittest.TestCase):
         self.assertEqual(result.get("primary_damage"), "Front End")
         self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
 
-    def test_parse_title_for_year_make_model(self) -> None:
         parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
         self.assertEqual(parsed["year"], "2014")
         self.assertEqual(parsed["make"], "TOYOTA")
         self.assertEqual(parsed["model"], "CAMRY")
 
-    def test_extract_image_urls_filters_other_vehicle(self) -> None:
-        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
-        payloads = [
-            {
-                "imageUrls": [
-                    "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-                    "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
-                ]
-            }
-        ]
-        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
-        self.assertEqual(len(urls), 1)
-        self.assertIn("45089484", urls[0])
-
-    def test_extract_image_urls_deduplicates_same_url(self) -> None:
+    def test_extract_image_urls_filters_and_deduplicates(self) -> None:
         vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         payloads = [{"imageUrls": [
             "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
             "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
         ]}]
         urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
         self.assertEqual(len(urls), 1)
+        self.assertIn("45089484", urls[0])
 
-    def test_dom_hints_detect_captcha_and_antibot(self) -> None:
+    def test_dom_hints_and_access_notes_detect_antibot(self) -> None:
         hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
         self.assertTrue(hints["has_captcha_text"])
         self.assertTrue(hints["has_antibot_text"])
 
-    def test_access_notes_reflect_antibot_signals(self) -> None:
         summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
         notes = self.parser._build_access_notes(summary, [])
         self.assertTrue(notes["possible_captcha"])

tests/test_scraper.py

@@ -6,6 +6,8 @@ from unittest.mock import MagicMock
 
 from iaai_scraper.core.config import Settings
 from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from iaai_scraper.core.exceptions import ListingResumeError
+from iaai_scraper.discovery import SitemapDiscoveryError
 from iaai_scraper.scraper import IAAIScraper
 from iaai_scraper.storage.schemas import CarRecord
 
@@ -28,104 +30,206 @@ class TestScraperSync(unittest.TestCase):
         s.database.url = "sqlite://"
         return IAAIScraper(s)
 
-    def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
+    def test_sync_vehicle_uses_db_record(self) -> None:
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=1)
         scraper.persistence.finish_sync_run = MagicMock()
         scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-
         scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
 
         result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
 
         self.assertEqual(result["status"], "success")
         self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
+        scraper.persistence.upsert_car.assert_called_once()
 
-    def test_sync_listing_uses_db_record_without_remapping(self) -> None:
+    def test_only_new_routing_legacy_and_streaming(self) -> None:
+        # Legacy path: only_new без listing_url.
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=2)
         scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set()))
-
-        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
-        scraper.sync_batch = MagicMock(return_value={
-            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
+            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
+        ))
+        scraper.collect_listing = MagicMock(return_value={
+            "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
+            ]
         })
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
+        })
+        result = scraper.sync_listing(only_new=True)
+        self.assertEqual(result["skipped_existing"], 2)
+        self.assertEqual(result["cars_upserted"], 1)
+
+        # Streaming path: only_new + listing_url.
+        scraper2 = self._make_scraper()
+        scraper2.persistence.create_tables = MagicMock()
+        scraper2.persistence.start_sync_run = MagicMock(return_value=6)
+        scraper2.persistence.finish_sync_run = MagicMock()
+        scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used"))
+        scraper2._sync_listing_streaming = MagicMock(return_value={
+            "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False},
+            "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0,
+            "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(),
+        })
+        result2 = scraper2.sync_listing(
+            only_new=True,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            year_min=2020, year_max=2027,
+        )
+        self.assertEqual(result2["cars_upserted"], 10)
+        scraper2._sync_listing_streaming.assert_called_once()
+        scraper2.collect_listing.assert_not_called()
+
+    def test_full_scan_uses_sitemap_discovery_path(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.discovery.mode = "listing"
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=88)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
+        scraper._sync_listing_via_sitemap = MagicMock(return_value={
+            "listing": {
+                "vehicles_collected": 123,
+                "early_stopped": False,
+                "truncated_by_time_budget": False,
+                "pagination_interrupted": False,
+                "source": "sitemap",
+            },
+            "total": 123,
+            "skipped_existing": 0,
+            "cars_upserted": 120,
+            "cars_failed": 3,
+            "images_upserted": 500,
+            "failures": [{"vehicle_url": "v", "error": "e"}],
+            "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"},
+        })
+        scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used"))
 
         result = scraper.sync_listing()
 
-        self.assertEqual(result["cars_upserted"], 1)
-        self.assertEqual(result["cars_failed"], 0)
-        self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
-        scraper.sync_batch.assert_called_once()
+        scraper._sync_listing_via_sitemap.assert_called_once()
+        scraper._sync_listing_streaming.assert_not_called()
+        self.assertEqual(result["cars_upserted"], 120)
+        self.assertTrue(result["full_scan_completed"])
 
-    def test_sync_listing_respects_limit(self) -> None:
+    def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None:
         scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=89)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
+        scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down"))
+        result = scraper.sync_listing()
 
+        scraper._sync_listing_via_sitemap.assert_called_once()
+        self.assertEqual(result["status"], "failed")
+        self.assertEqual(result["cars_upserted"], 0)
+
+    def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
+        scraper = self._make_scraper()
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=3)
         scraper.persistence.finish_sync_run = MagicMock()
 
-        # Проверка пути only_new с limit.
-        scraper._collect_listing_iterative = MagicMock(return_value=(
-            ["https://www.iaai.com/VehicleDetail/222~US"],
-            ["https://www.iaai.com/VehicleDetail/222~US",
-             "https://www.iaai.com/VehicleDetail/333~US"],
-            {"vehicle_urls": [], "pages_collected": 1, "early_stopped": False},
-            0,
-        ))
-        scraper.sync_batch = MagicMock(return_value={
-            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
-        })
+        call_args_log: list[dict] = []
+        def _fake_sync_listing(**kwargs):
+            call_args_log.append(kwargs)
+            return {
+                "status": "success", "cars_upserted": 5, "cars_failed": 0,
+                "images_upserted": 10, "skipped_existing": 0,
+                "listing": {"vehicles_collected": 50}, "failures": [],
+            }
 
-        scraper.sync_listing(limit=1)
+        scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing)
+        segments = [
+            {"make": "TOYOTA", "year_min": 2020, "year_max": 2027},
+            {"make": "FORD", "year_min": None, "year_max": None},
+            {"make": "HONDA", "year_min": None, "year_max": None},
+        ]
 
-        # Должен уйти только один URL.
-        scraper.sync_batch.assert_called_once()
-        batch_urls = scraper.sync_batch.call_args[0][0]
-        self.assertEqual(len(batch_urls), 1)
+        # Resume: пропускаем TOYOTA, начинаем сразу с FORD.
+        result = scraper.sync_listing_segmented(
+            segments=segments, start_segment=1,
+        )
 
-    def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
+        self.assertEqual(result["segments_completed"], 2)  # FORD + HONDA
+        self.assertEqual(result["cars_upserted"], 10)
+        # URL содержит бренд.
+        self.assertIn("FORD", call_args_log[0]["listing_url"])
+        self.assertIn("HONDA", call_args_log[1]["listing_url"])
+
+    def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None:
         scraper = self._make_scraper()
+        scraper.sync_listing = MagicMock(side_effect=[
+            {
+                "status": "failed",
+                "full_scan_completed": False,
+                "cars_upserted": 0,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 0},
+                "failures": [{"vehicle_url": "segment", "error": "resume failed"}],
+            },
+            {
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 10},
+                "failures": [],
+            },
+        ])
 
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=5)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
-            {"https://www.iaai.com/VehicleDetail/111~US"},
-            {"iaai:222"},
-        ))
-
-        scraper.collect_listing = MagicMock(return_value={
-            "vehicle_urls": [
-                "https://www.iaai.com/VehicleDetail/111~US",  # exists by URL
-                "https://www.iaai.com/VehicleDetail/222~US",  # exists by ID
-                "https://www.iaai.com/VehicleDetail/333~US",  # new
+        result = scraper.sync_listing_segmented(
+            segments=[
+                {"make": "EAGLE", "year_min": None, "year_max": None},
+                {"make": "FORD", "year_min": None, "year_max": None},
             ]
-        })
-        # Возвращаем результат для одного нового авто.
-        scraper.sync_batch = MagicMock(return_value={
-            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
-        })
+        )
 
-        result = scraper.sync_listing(only_new=True)
+        self.assertFalse(result["full_scan_completed"])
+        self.assertEqual(result["status"], "partial_success")
 
-        self.assertEqual(result["skipped_existing"], 2)
-        self.assertEqual(result["cars_upserted"], 1)
-        # В batch должен попасть только новый URL.
-        scraper.sync_batch.assert_called_once()
-        batch_urls = scraper.sync_batch.call_args[0][0]
-        self.assertEqual(len(batch_urls), 1)
-        self.assertIn("333", batch_urls[0])
-        scraper.persistence.get_existing_urls_and_ids.assert_called_once()
+    def test_build_segment_listing_url(self) -> None:
+        base = "https://www.iaai.com/Vehiclelisting/Cars"
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
+        )
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
+
+    def test_guard_and_protection_detection(self) -> None:
+        with self.assertRaises(AntiBotDetectedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
+                 "access_notes": {}, "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        with self.assertRaises(SiteStructureChangedError):
+            IAAIScraper._raise_if_blocked_or_incomplete(
+                {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
+                 "access_notes": {"possible_captcha": False, "possible_antibot": False},
+                 "vehicle_summary": {}},
+                "https://www.iaai.com/VehicleDetail/999~US",
+            )
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
 
     def test_close_resets_browser_state(self) -> None:
         scraper = self._make_scraper()
@@ -134,85 +238,202 @@ class TestScraperSync(unittest.TestCase):
         scraper.context = MagicMock()
         scraper.browser = MagicMock()
         scraper.playwright = MagicMock()
-
         scraper.close()
-
         http_pool.clear.assert_called_once()
-        self.assertIsNone(scraper._http_pool)
         self.assertIsNone(scraper.context)
         self.assertIsNone(scraper.browser)
-        self.assertIsNone(scraper.playwright)
 
-    def test_guard_raises_on_antibot_signals(self) -> None:
-        with self.assertRaises(AntiBotDetectedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
-                {
-                    "dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
-                    "access_notes": {},
-                    "vehicle_summary": {},
-                },
-                "https://www.iaai.com/VehicleDetail/999~US",
-            )
-
-    def test_guard_raises_on_empty_vehicle_page(self) -> None:
-        with self.assertRaises(SiteStructureChangedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
-                {
-                    "dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
-                    "access_notes": {"possible_captcha": False, "possible_antibot": False},
-                    "vehicle_summary": {},
-                },
-                "https://www.iaai.com/VehicleDetail/999~US",
-            )
-
-    def test_is_protection_or_network_error_detects_known_signals(self) -> None:
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
-        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
-
-    def test_recover_empty_listing_page_reload_recovers_links(self) -> None:
+    def test_recover_empty_listing_page(self) -> None:
         scraper = self._make_scraper()
         page = MagicMock()
         page_result = SimpleNamespace(
             vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
         )
         scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper.listing_collector.open_cars_listing = MagicMock()
 
-        all_raw_urls: list[str] = []
-        seen_urls: set[str] = set()
-
-        recovered_result, recovered_urls = scraper._recover_empty_listing_page(
-            page,
-            page_number=5,
-            all_raw_urls=all_raw_urls,
-            seen_urls=seen_urls,
+        # Страница > 1: reload.
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=5, all_raw_urls=[], seen_urls=set(),
         )
-
         page.reload.assert_called_once()
-        self.assertIs(recovered_result, page_result)
-        self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/123~US"])
-        self.assertEqual(all_raw_urls, ["https://www.iaai.com/VehicleDetail/123~US"])
+        self.assertEqual(len(urls), 1)
 
-    def test_recover_empty_listing_page_reopens_listing_for_first_page(self) -> None:
+        # Страница 1: переоткрытие листинга.
+        page.reset_mock()
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=1, all_raw_urls=[], seen_urls=set(),
+        )
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        page.reload.assert_not_called()
+
+    def test_sync_listing_always_starts_from_page_one(self) -> None:
         scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1
+
         page = MagicMock()
         page_result = SimpleNamespace(
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/456~US")],
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            next_page_detected=False,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        # Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
+        scraper._reopen_listing_and_resume = MagicMock(
+            side_effect=AssertionError("pagination resume must not be used")
         )
         scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
         scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
 
-        recovered_result, recovered_urls = scraper._recover_empty_listing_page(
-            page,
-            page_number=1,
-            all_raw_urls=[],
-            seen_urls=set(),
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
         )
 
-        scraper.listing_collector.open_cars_listing.assert_called_once_with(page)
-        page.reload.assert_not_called()
-        self.assertIs(recovered_result, page_result)
-        self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/456~US"])
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        scraper._reopen_listing_and_resume.assert_not_called()
+        scraper.sync_batch.assert_called_once()
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(result["total"], 1)
+
+    def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1000
+
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            next_page_detected=True,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper.listing_collector.go_to_next_page = MagicMock(return_value=False)
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed"))
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 0,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+
+        self.assertTrue(result["listing"]["pagination_interrupted"])
+
+    def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1000
+
+        page = MagicMock()
+        first_page = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")],
+            next_page_detected=False,
+        )
+        second_page = SimpleNamespace(
+            page_number=2,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")],
+            next_page_detected=False,
+        )
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page])
+        scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False])
+        scraper._extract_page_urls = MagicMock(side_effect=[
+            ["https://www.iaai.com/VehicleDetail/111~US"],
+            ["https://www.iaai.com/VehicleDetail/222~US"],
+        ])
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 2,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+        )
+
+        self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2)
+        scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2)
+        self.assertEqual(result["listing"]["pages_collected"], 2)
+
+    def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.create_tables = MagicMock()
+        scraper.persistence.start_sync_run = MagicMock(return_value=77)
+        scraper.persistence.finish_sync_run = MagicMock()
+        scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
+        scraper._sync_listing_streaming = MagicMock(return_value={
+            "listing": {
+                "vehicles_collected": 10,
+                "early_stopped": False,
+                "truncated_by_time_budget": False,
+                "pagination_interrupted": True,
+            },
+            "total": 10,
+            "skipped_existing": 0,
+            "cars_upserted": 10,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+            "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"},
+        })
+
+        result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA")
+
+        self.assertTrue(result["full_scan_completed"] is False)
+        scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
 
 
 if __name__ == "__main__":

tests/test_sitemap_discovery.py

@@ -0,0 +1,217 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import patch
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.discovery.sitemap import (
+    SitemapBlockedError,
+    SitemapDiscoveryError,
+    SitemapFetchResult,
+    _filter_vehicle_urls,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+
+class TestSitemapDiscovery(unittest.TestCase):
+    @staticmethod
+    def _fetch_result(url: str, payload: bytes, *, source: str = "curl_cffi") -> SitemapFetchResult:
+        return SitemapFetchResult(url=url, payload=payload, source=source)
+
+    def test_filter_vehicle_urls_dedupes_and_normalizes(self) -> None:
+        urls = _filter_vehicle_urls([
+            "https://www.iaai.com/VehicleDetail/111~US?foo=1",
+            "https://www.iaai.com/VehicleDetail/111~US?bar=2",
+            "https://www.iaai.com/VehicleDetail/222~US",
+            "https://www.iaai.com/about",
+        ])
+
+        self.assertEqual(
+            urls,
+            [
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+            ],
+        )
+
+    def test_discover_vehicle_urls_from_sitemap(self) -> None:
+        index_xml = b"""
+        <sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <sitemap><loc>https://www.iaai.com/sitemap-a.xml</loc></sitemap>
+          <sitemap><loc>https://www.iaai.com/sitemap-b.xml</loc></sitemap>
+                    <sitemap><loc>https://www.iaai.com/sitemapauctions1.xml</loc></sitemap>
+        </sitemapindex>
+        """
+        sitemap_a = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+          <url><loc>https://www.iaai.com/VehicleDetail/222~US?x=1</loc></url>
+        </urlset>
+        """
+        sitemap_b = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/222~US?y=2</loc></url>
+          <url><loc>https://www.iaai.com/VehicleDetail/333~US</loc></url>
+        </urlset>
+        """
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap-a.xml"):
+                return self._fetch_result(url, sitemap_a)
+            if url.endswith("sitemap-b.xml"):
+                return self._fetch_result(url, sitemap_b)
+            raise AssertionError(f"unexpected url: {url}")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(
+            result,
+            [
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
+            ],
+        )
+
+    def test_discover_vehicle_urls_raises_on_empty_index(self) -> None:
+        empty_index = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
+        with patch(
+            "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
+            return_value=self._fetch_result("https://www.iaai.com/sitemap_index.xml", empty_index),
+        ):
+            with self.assertRaises(SitemapDiscoveryError):
+                discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+    def test_discover_vehicle_urls_skips_malformed_and_non_vehicle_sitemaps(self) -> None:
+        sitemap_vehicle = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+            <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+        sitemap_non_vehicle = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+            <url><loc>https://www.iaai.com/SalesList/111~US/04222026</loc></url>
+        </urlset>
+        """
+        index_xml = b"""
+        <sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+            <sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>
+            <sitemap><loc>https://www.iaai.com/sitemapbranches1.xml</loc></sitemap>
+            <sitemap><loc>https://www.iaai.com/sitemap2.xml</loc></sitemap>
+            <sitemap><loc>https://www.iaai.com/sitemap3.xml</loc></sitemap>
+        </sitemapindex>
+        """
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap_vehicle)
+            if url.endswith("sitemap2.xml"):
+                return self._fetch_result(url, sitemap_non_vehicle)
+            if url.endswith("sitemap3.xml"):
+                raise SitemapDiscoveryError("broken sitemap")
+            raise AssertionError(f"unexpected url: {url}")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
+
+    def test_discover_vehicle_urls_falls_back_to_regex_loc_extraction(self) -> None:
+        malformed_index = (
+            b"<sitemapindex>"
+            b"<sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>"
+            b"<broken"
+        )
+        sitemap_vehicle = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, malformed_index)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap_vehicle)
+            raise AssertionError(f"unexpected url: {url}")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
+
+    def test_discovery_uses_direct_probe_when_index_has_no_urls(self) -> None:
+        index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
+        sitemap_one = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+
+        settings = Settings()
+        settings.discovery.sitemap_direct_probe_limit = 2
+        settings.discovery.sitemap_direct_probe_stop_after_misses = 1
+
+        def _fake_fetch(url: str) -> SitemapFetchResult:
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap_one)
+            raise SitemapDiscoveryError("not found")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap_with_stats(
+                "https://www.iaai.com/sitemap_index.xml",
+                settings=settings,
+            )
+
+        self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
+        self.assertEqual(result.stats.direct_probe_hits, 1)
+
+    def test_discovery_stats_report_direct_probe_hits(self) -> None:
+        index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
+        sitemap1 = b"""
+        <urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
+          <url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
+        </urlset>
+        """
+
+        def _fake_fetch(url: str):
+            if url.endswith("sitemap_index.xml"):
+                return self._fetch_result(url, index_xml)
+            if url.endswith("sitemap1.xml"):
+                return self._fetch_result(url, sitemap1)
+            raise SitemapDiscoveryError("not found")
+
+        with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
+            result = discover_vehicle_urls_from_sitemap_with_stats("https://www.iaai.com/sitemap_index.xml")
+
+        self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
+        self.assertEqual(result.stats.transport, "curl_cffi")
+        self.assertEqual(result.stats.direct_probe_hits, 1)
+        self.assertGreaterEqual(result.stats.direct_probe_misses, 1)
+
+    def test_block_page_raises_discovery_error(self) -> None:
+        settings = Settings()
+        settings.discovery.sitemap_direct_probe_limit = 1
+        settings.discovery.sitemap_direct_probe_stop_after_misses = 1
+
+        with patch(
+            "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
+            side_effect=SitemapBlockedError("Anti-bot page returned for https://www.iaai.com/sitemap_index.xml"),
+        ):
+            with self.assertRaises(SitemapDiscoveryError):
+                discover_vehicle_urls_from_sitemap(
+                    "https://www.iaai.com/sitemap_index.xml",
+                    settings=settings,
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_utils.py

@@ -3,28 +3,72 @@ from __future__ import annotations
 import unittest
 
 from iaai_scraper.core.utils import deep_find_key
+from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
 
 
 class TestDeepFindKey(unittest.TestCase):
-    def test_finds_key_in_nested_structure(self) -> None:
+    def test_finds_nested_and_respects_depth(self) -> None:
         payload = {
             "root": {
                 "target": "a",
                 "nested": [{"target": "b"}, {"x": 1}],
             }
         }
+        self.assertEqual(deep_find_key(payload, {"target"}), ["a", "b"])
 
-        found = deep_find_key(payload, {"target"})
-        self.assertEqual(found, ["a", "b"])
+        deep_payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=2), [])
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"])
 
-    def test_respects_max_depth(self) -> None:
-        payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
 
-        found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2)
-        found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8)
+class TestParseListingSegments(unittest.TestCase):
+    def test_empty_and_invalid_return_empty(self) -> None:
+        self.assertEqual(parse_listing_segments(""), [])
+        self.assertEqual(parse_listing_segments("  "), [])
+        self.assertEqual(parse_listing_segments("invalid"), [])
 
-        self.assertEqual(found_too_shallow, [])
-        self.assertEqual(found_enough_depth, ["value"])
+    def test_auto_segments_complete_coverage(self) -> None:
+        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
+        segs = parse_listing_segments("auto")
+
+        # Точное число сегментов.
+        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
+        self.assertEqual(len(segs), expected)
+
+        # Все бренды из списка присутствуют.
+        makes_in_segments = {s["make"] for s in segs}
+        for make in IAAI_DEFAULT_MAKES:
+            self.assertIn(make, makes_in_segments)
+
+        # Крупные бренды разбиты на 3 сегмента с годами.
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        self.assertEqual(len(toyota), 3)
+        for s in toyota:
+            self.assertIsNotNone(s["year_min"])
+
+        # Мелкие бренды без годов.
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+
+        # Годовые диапазоны покрывают 1950-2027.
+        years = set()
+        for yr_min, yr_max in _YEAR_SPLITS:
+            years.update(range(yr_min, yr_max + 1))
+        for year in range(1950, 2027):
+            self.assertIn(year, years)
+
+    def test_json_input_formats(self) -> None:
+        # Массив строк.
+        segs = parse_listing_segments('["toyota", "ford"]')
+        self.assertEqual(len(segs), 2)
+        self.assertEqual(segs[0]["make"], "TOYOTA")
+
+        # Массив объектов с годами.
+        segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]')
+        self.assertEqual(segs[0]["make"], "BMW")
+        self.assertEqual(segs[0]["year_min"], 2020)
+        self.assertEqual(segs[0]["year_max"], 2025)
 
 
 if __name__ == "__main__":

tests/test_worker_tasks.py

@@ -1,39 +1,231 @@
 from __future__ import annotations
 
-import json
 import unittest
 from unittest.mock import MagicMock, patch
 
 from iaai_scraper.worker import tasks
 
 
+def make_settings_stub(*, parallel_segments: bool = False) -> MagicMock:
+    settings_stub = MagicMock()
+    settings_stub.celery.parallel_segments = parallel_segments
+    settings_stub.celery.task_soft_time_limit = 3300
+    settings_stub.celery.task_time_limit = 3600
+    settings_stub.celery.task_stall_timeout_seconds = 600
+    settings_stub.listing.listing_segments_json = "ignored"
+    settings_stub.discovery.mode = "listing"
+    settings_stub.discovery.hourly_mode = "rolling_refresh"
+    return settings_stub
+
+
 class TestWorkerTaskLockHelpers(unittest.TestCase):
-    def test_acquire_lock_returns_true_on_success(self) -> None:
+    def test_sync_listing_task_uses_hourly_sitemap_even_when_mode_is_not_sitemap(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "listing"
+        settings_stub.discovery.hourly_mode = "rolling_refresh"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 25,
+                 "cars_failed": 0,
+                 "images_upserted": 50,
+                 "skipped_existing": 100,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_rolling_refresh",
+                 "discovered_urls": 105,
+                 "new_urls": 5,
+                 "refresh_urls": 20,
+                 "sold_marked": 2,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-hourly-force-sitemap")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
+        hourly_sync.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_uses_hourly_sitemap_rolling_refresh_after_bootstrap(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "sitemap"
+        settings_stub.discovery.hourly_mode = "rolling_refresh"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 25,
+                 "cars_failed": 0,
+                 "images_upserted": 50,
+                 "skipped_existing": 100,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_rolling_refresh",
+                 "discovered_urls": 105,
+                 "new_urls": 5,
+                 "refresh_urls": 20,
+                 "sold_marked": 2,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-hourly")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
+        hourly_sync.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_can_use_hourly_sitemap_diff_when_configured(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "sitemap"
+        settings_stub.discovery.hourly_mode = "diff"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_hourly_sitemap_diff_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 5,
+                 "cars_failed": 0,
+                 "images_upserted": 10,
+                 "skipped_existing": 100,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_diff",
+                 "discovered_urls": 105,
+                 "new_urls": 5,
+                 "sold_marked": 2,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            tasks.sync_listing_task.push_request(id="task-hourly-diff")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_diff")
+        hourly_sync.assert_called_once()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_forces_sitemap_full_scan_in_bootstrap(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        settings_stub.discovery.mode = "listing"
+
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[{"make": "HONDA"}]):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 99,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 10,
+                "cars_failed": 0,
+                "images_upserted": 20,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            sync_listing_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
+            scraper = MagicMock()
+            scraper.sync_listing = sync_listing_mock
+            scraper.sync_listing_segmented = sync_listing_segmented_mock
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value = scraper
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-bootstrap-sitemap")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        sync_listing_mock.assert_called_once_with(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            only_new=False,
+        )
+        sync_listing_segmented_mock.assert_not_called()
+        release_lock.assert_called_once()
+
+    def test_lock_acquire_refresh_release(self) -> None:
         redis_client = MagicMock()
+
+        # Acquire.
         redis_client.set.return_value = True
-
-        acquired = tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)
-
-        self.assertTrue(acquired)
+        self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
         redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
 
-    def test_refresh_lock_if_owner_extends_ttl(self) -> None:
-        redis_client = MagicMock()
+        # Refresh.
         redis_client.eval.return_value = 1
+        self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
 
-        refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)
-
-        self.assertTrue(refreshed)
-        redis_client.eval.assert_called_once()
-
-    def test_release_lock_if_owner_uses_owner_token(self) -> None:
-        redis_client = MagicMock()
-
+        # Release.
+        redis_client.eval.reset_mock()
         tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
-
-        redis_client.eval.assert_called_once()
         args = redis_client.eval.call_args[0]
-        self.assertEqual(args[1], 1)
         self.assertEqual(args[2], "lock:key")
         self.assertEqual(args[3], "owner-token")
 
@@ -62,6 +254,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_is_full_scan_done", return_value=True), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
              patch.object(tasks, "_run_browser_job", return_value={
                  "run_id": 7,
                  "cars_upserted": 2,
@@ -90,7 +283,8 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         heartbeat_thread.join.assert_called_once()
         release_lock.assert_called_once()
 
-    def test_clear_orphan_sync_listing_lock_deletes_when_no_tasks_running(self) -> None:
+    def test_clear_orphan_sync_listing_lock(self) -> None:
+        # Нет запущенных задач → удаляет.
         redis_client = MagicMock()
         redis_client.get.return_value = "owner-token"
         redis_client.ttl.return_value = 120
@@ -101,27 +295,21 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         inspector.scheduled.return_value = {"worker@node": []}
         celery_app.control.inspect.return_value = inspector
 
-        cleared = tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)
-
-        self.assertTrue(cleared)
+        self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
         redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
 
-    def test_clear_orphan_sync_listing_lock_keeps_when_task_detected(self) -> None:
-        redis_client = MagicMock()
-        redis_client.get.return_value = "owner-token"
-        celery_app = MagicMock()
-        inspector = MagicMock()
-        inspector.active.return_value = {
-            "worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]
-        }
-        inspector.reserved.return_value = {"worker@node": []}
-        inspector.scheduled.return_value = {"worker@node": []}
-        celery_app.control.inspect.return_value = inspector
+        # Задача активна → не удаляет.
+        redis_client2 = MagicMock()
+        redis_client2.get.return_value = "owner-token"
+        inspector2 = MagicMock()
+        inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
+        inspector2.reserved.return_value = {"worker@node": []}
+        inspector2.scheduled.return_value = {"worker@node": []}
+        celery_app2 = MagicMock()
+        celery_app2.control.inspect.return_value = inspector2
 
-        cleared = tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)
-
-        self.assertFalse(cleared)
-        redis_client.delete.assert_not_called()
+        self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
+        redis_client2.delete.assert_not_called()
 
     def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
         with patch.object(tasks, "_get_persistence") as get_persistence, \
@@ -131,6 +319,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_is_full_scan_done", return_value=True), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
              patch.object(tasks, "_run_browser_job", return_value={
                  "run_id": 9,
                  "cars_upserted": 3,
@@ -159,28 +348,191 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         self.assertEqual(acquire_lock.call_count, 2)
         release_lock.assert_called_once()
 
-    def test_sync_listing_checkpoint_roundtrip(self) -> None:
-        redis_client = MagicMock()
+    def test_checkpoint_save_load_roundtrip(self) -> None:
         storage: dict[str, str] = {}
-        redis_client.set.side_effect = lambda key, value: storage.__setitem__(key, value)
+
+        def _fake_set(key, value, ex=None):
+            storage[key] = value
+            return True
+
+        redis_client = MagicMock()
+        redis_client.set.side_effect = _fake_set
         redis_client.get.side_effect = lambda key: storage.get(key)
 
-        tasks._save_sync_checkpoint(
-            redis_client,
-            task_id="task-1",
-            page_number=12,
+        tasks._save_last_completed_segment(redis_client, 12)
+
+        self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
+        self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
+
+    def test_load_checkpoint_clears_invalid_payload(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = "{not-a-number"
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
+
+    def test_load_checkpoint_empty_when_missing(self) -> None:
+        redis_client = MagicMock()
+        redis_client.get.return_value = None
+
+        self.assertIsNone(tasks._load_last_completed_segment(redis_client))
+        redis_client.delete.assert_not_called()
+
+    def test_sync_listing_bootstrap_prefers_sitemap_over_segment_resume(self) -> None:
+        segments = [
+            {"make": "ACURA"},
+            {"make": "AUDI"},
+            {"make": "BMW"},
+            {"make": "EAGLE"},
+        ]
+        settings_stub = make_settings_stub()
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 11, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-resume-seg")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        sync_listing_mock.assert_called_once_with(
             make=None,
             model=None,
             lane="iaai_cars",
+            limit=None,
+            only_new=False,
         )
+        sync_segmented_mock.assert_not_called()
+        release_lock.assert_called_once()
 
-        checkpoint = tasks._load_sync_checkpoint(redis_client)
+    def test_sync_listing_hourly_path_ignores_checkpoint_after_full_scan_completed(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        settings_stub = make_settings_stub()
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
+                 "status": "success",
+                 "run_id": None,
+                 "cars_upserted": 1,
+                 "cars_failed": 0,
+                 "images_upserted": 0,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": None,
+                 "failures": [],
+                 "hourly_mode": "sitemap_rolling_refresh",
+                 "discovered_urls": 10,
+                 "new_urls": 1,
+                 "refresh_urls": 1,
+                 "sold_marked": 0,
+             }) as hourly_sync, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            # Оставшийся чекпоинт не должен использоваться.
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
 
-        self.assertIsNotNone(checkpoint)
-        self.assertEqual(checkpoint["last_successful_page"], 12)
-        self.assertEqual(checkpoint["status"], "in_progress")
+            tasks.sync_listing_task.push_request(id="task-792")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
 
-    def test_sync_listing_task_resumes_from_checkpoint_page(self) -> None:
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
+        hourly_sync.assert_called_once()
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_bootstrap_ignores_beyond_segment_checkpoint(self) -> None:
+        segments = [{"make": "ACURA"}, {"make": "AUDI"}]
+        settings_stub = make_settings_stub()
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "Settings", return_value=settings_stub), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 15, "status": "success", "full_scan_completed": True,
+                "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-beyond")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        sync_listing_mock.assert_called_once_with(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            only_new=False,
+        )
+        sync_segmented_mock.assert_not_called()
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
              patch.object(tasks, "_acquire_lock", return_value=True), \
@@ -188,48 +540,121 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
-             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()):
-            persistence = MagicMock()
-            get_persistence.return_value = persistence
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 13,
+                 "status": "partial_success",
+                 "full_scan_completed": True,
+                 "cars_upserted": 2,
+                 "cars_failed": 1,
+                 "images_upserted": 3,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 4.0,
+                 "failures": [{"vehicle_url": "v", "error": "e"}],
+             }), \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
-            redis_client.get.side_effect = lambda key: json.dumps({
-                "status": "in_progress",
-                "last_successful_page": 9,
-                "make": None,
-                "model": None,
-                "lane": "iaai_cars",
-            }) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            redis_client.get.return_value = None
             get_redis.return_value = redis_client
-            stop_event = MagicMock()
-            heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
 
-            sync_listing_mock = MagicMock(return_value={
-                "run_id": 11,
-                "status": "success",
-                "full_scan_completed": True,
-                "cars_upserted": 1,
-                "cars_failed": 0,
-                "images_upserted": 0,
-                "skipped_existing": 0,
-                "elapsed_seconds": 1.0,
-                "failures": [],
-            })
-            scraper_ctx = MagicMock()
-            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
-            scraper_ctx.__exit__.return_value = None
+            tasks.sync_listing_task.push_request(id="task-791")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
 
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
-                tasks.sync_listing_task.push_request(id="task-789")
+        self.assertEqual(result["status"], "partial_success")
+        # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
+        # до браузерного job + после. Главное — вызов произошёл.
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
+    def test_try_set_followup_pending_deduplicates(self) -> None:
+        redis_client = MagicMock()
+        redis_client.set.side_effect = [True, False]
+
+        self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+        self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
+
+    def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="max_retries_exceeded",
+        )
+
+        self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
+        self.assertFalse(should_enqueue)
+        redis_client.expire.assert_called_once_with(
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
+            tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
+        )
+
+    def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
+        redis_client = MagicMock()
+        redis_client.incr.return_value = 1
+
+        streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
+            redis_client,
+            reason="bootstrap_not_completed",
+        )
+
+        self.assertEqual(streak, 1)
+        self.assertTrue(should_enqueue)
+
+    def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
+        settings_stub = make_settings_stub(parallel_segments=False)
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+            patch.object(tasks, "_get_redis") as get_redis, \
+            patch.object(tasks, "Settings", return_value=settings_stub), \
+            patch.object(tasks, "_acquire_lock", return_value=True), \
+            patch.object(tasks, "_is_full_scan_done", return_value=False), \
+            patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+            patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+            patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+            patch.object(
+                tasks,
+                "_bump_bootstrap_failure_streak",
+                return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False),
+            ) as bump_streak, \
+            patch.object(tasks, "_clear_followup_pending") as clear_pending, \
+            patch.object(tasks.sync_listing_task, "update_state"), \
+            patch.object(
+                tasks,
+                "_run_browser_job",
+                return_value={
+                    "run_id": 99,
+                    "status": "failed",
+                    "full_scan_completed": False,
+                    "cars_upserted": 0,
+                    "cars_failed": 0,
+                    "images_upserted": 0,
+                    "skipped_existing": 0,
+                    "elapsed_seconds": 1.0,
+                    "failures": [{"vehicle_url": "listing", "error": "bad resume"}],
+                    "listing": {"vehicles_collected": 0},
+                },
+            ):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-900")
                 try:
                     result = tasks.sync_listing_task.run()
                 finally:
                     tasks.sync_listing_task.pop_request()
 
-        self.assertEqual(result["status"], "success")
-        self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 10)
-        clear_checkpoint.assert_called()
-        release_lock.assert_called_once()
+        self.assertEqual(result["status"], "failed")
+        bump_streak.assert_called_once()
+        clear_pending.assert_called()
+        task_app.send_task.assert_not_called()
 
 
 if __name__ == "__main__":