fix docker scraping

improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
2026-04-13 16:35:08 +03:00
parent b9557922ed
commit a4c93a1df1
30 changed files with 2239 additions and 624 deletions
--- a/.env.example
+++ b/.env.example
@@ -10,25 +10,25 @@ IAAI_MAX_CAPTURED_JSON_RESPONSES=30
 # Sequential listing-first mode.
 IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
 IAAI_MAX_PAGES_PER_RUN=10
-IAAI_MAX_VEHICLES_PER_RUN=30
-IAAI_PAGE_LINK_LIMIT=100
+IAAI_MAX_VEHICLES_PER_RUN=50000
+IAAI_PAGE_LINK_LIMIT=500
 IAAI_INCLUDE_PAGINATION=true
 IAAI_COLLECT_CURRENT_PAGE_ONLY=false

-# Human-like pacing.
-IAAI_HUMAN_PACE_ENABLED=true
-IAAI_AFTER_LISTING_OPEN_MIN_S=2.5
-IAAI_AFTER_LISTING_OPEN_MAX_S=4.5
-IAAI_AFTER_FILTER_ACTION_MIN_S=2.0
-IAAI_AFTER_FILTER_ACTION_MAX_S=4.0
-IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5
-IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0
-IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0
-IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5
-IAAI_BETWEEN_VEHICLES_MIN_S=3.0
-IAAI_BETWEEN_VEHICLES_MAX_S=6.0
-IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0
-IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0
+# Паузы (отключены для максимальной скорости; включи на VPS если попадаешь под блокировки).
+IAAI_HUMAN_PACE_ENABLED=false
+IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
+IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
+IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
+IAAI_BETWEEN_VEHICLES_MIN_S=0.02
+IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5

 # Sync settings
 IAAI_SYNC_ONLY_NEW=true
@@ -40,16 +40,6 @@ IAAI_RETRY_DELAY_SECONDS=2.5
 IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
 IAAI_RETRY_JITTER_SECONDS=0.25

-# Session persistence (cookies + localStorage)
-# Keeps browser session alive between runs (up to 30 days).
-IAAI_STORAGE_STATE_PATH=storage_state.json
-IAAI_SESSION_MAX_AGE_DAYS=30
-IAAI_SESSION_SAVE_ON_EXIT=true
-
-# IAAI login credentials (for authenticated scraping)
-# IAAI_LOGIN_EMAIL=your@email.com
-# IAAI_LOGIN_PASSWORD=your_password
-
 # Proxy (HTTP/HTTPS preferred for Playwright)
 # Chromium does not support SOCKS5 proxy authentication directly.
 # Use residential or mobile USA proxy.
@@ -60,8 +50,8 @@ IAAI_SESSION_SAVE_ON_EXIT=true
 # Database (PostgreSQL).
 IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
 IAAI_DATABASE_ECHO=false
-IAAI_DATABASE_POOL_SIZE=5
-IAAI_DATABASE_MAX_OVERFLOW=10
+IAAI_DATABASE_POOL_SIZE=10
+IAAI_DATABASE_MAX_OVERFLOW=20

 # ─── Redis (Celery broker) ─────────────────────────────────
 IAAI_REDIS_URL=redis://redis:6379/0
--- a/7
+++ b/7
@@ -10,9 +10,14 @@ WORKDIR /app
 COPY pyproject.toml uv.lock ./
 RUN pip install --no-cache-dir uv \
    && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \
-    && pip install --no-cache-dir -r /tmp/requirements.txt
+    && pip install --no-cache-dir -r /tmp/requirements.txt \
+    && pip install --no-cache-dir playwright-stealth
+
+# Install Firefox browser (headless-friendly, bypasses Incapsula)
+RUN python -m playwright install firefox

 COPY . .
+RUN pip install --no-cache-dir -e .
 RUN python -m compileall -q iaai_scraper
 RUN chmod +x entrypoint.sh
 RUN chown -R app:app /app
--- a/README.md
+++ b/README.md
@@ -27,10 +27,12 @@ Docker **не нужен**. Данные хранятся в SQLite-файле `
 git clone <repo-url>
 cd iaai_scraper_project
 pip install -e .
-playwright install chromium
+playwright install firefox
 iaai init-db
 ```

+`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker).
+
 Готовый `.env` уже в репозитории и настроен на SQLite  ничего менять не нужно.

 ### Запуск парсера
@@ -157,6 +159,8 @@ iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US"
 iaai sync-listing --limit 10
 ```

+`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic.
+
 ## Структура

 ```text
@@ -267,8 +271,6 @@ uv run pytest -q
 - `only_new`
 - `limit`

-Так же  используются: `lane`, `only_new`, `limit`.
-
 ### Секция `filters`

 Поддерживаются поля:
--- a/alembic/versions/001_initial.py
+++ b/alembic/versions/001_initial.py
@@ -1,9 +1,4 @@
-"""Initial schema — cars, images, sync_runs
-
-Revision ID: 001_initial
-Revises:
-Create Date: 2026-04-08
-"""
+# Начальная схема: cars, images, sync_runs
 from typing import Sequence, Union

 from alembic import op
@@ -16,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None


 def upgrade() -> None:
-    # --- cars ---
+    # Таблица cars — аукционные машины с IAAI
    op.create_table(
        "cars",
        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
@@ -53,7 +48,7 @@ def upgrade() -> None:
    op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
    op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)

-    # --- images ---
+    # Таблица images — изображения машин
    op.create_table(
        "images",
        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
@@ -63,7 +58,7 @@ def upgrade() -> None:
        sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
    )

-    # --- sync_runs ---
+    # Таблица sync_runs — логирование синхронизаций
    op.create_table(
        "sync_runs",
        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
--- a/alembic/versions/002_add_indexes.py
+++ b/alembic/versions/002_add_indexes.py
@@ -1,9 +1,4 @@
-"""Add performance indexes for growing database
-
-Revision ID: 002_add_indexes
-Revises: 001_initial
-Create Date: 2026-04-09
-"""
+# Индексы производительности
 from typing import Sequence, Union

 from alembic import op
@@ -15,25 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None


 def upgrade() -> None:
-    # cars: ускорение фильтрации по бренду в API и статистике
    op.create_index("ix_cars_brand", "cars", ["brand"])
-
-    # cars: составной индекс бренд+модель для комбинированных фильтров
    op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
-
-    # cars: ускорение фильтрации по году (year_min/year_max)
    op.create_index("ix_cars_year", "cars", ["year"])
-
-    # cars: ускорение фильтрации по статусу продажи
    op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
-
-    # cars: ускорение сортировки ORDER BY last_seen_at DESC (пагинация)
    op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
-
-    # images: ускорение JOIN/DELETE по car_id (критично при upsert)
    op.create_index("ix_images_car_id", "images", ["car_id"])
-
-    # sync_runs: ускорение поиска stale runs по статусу
    op.create_index("ix_sync_runs_status", "sync_runs", ["status"])


--- a/alembic/versions/003_add_composite_indexes.py
+++ b/alembic/versions/003_add_composite_indexes.py
@@ -0,0 +1,38 @@
+# Индексы для масштабированной БД (20k+ записей)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "003_add_composite_indexes"
+down_revision: Union[str, None] = "002_add_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Ускоряет поиск при mark_sold операциях
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
+        "ON cars (origin_url) "
+        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
+    )
+
+    # Composite index для проверки дубликатов изображений
+    op.create_index(
+        "ix_images_car_id_fullres",
+        "images",
+        ["car_id", "fullres_image"],
+    )
+
+    # Index для быстрого поиска existing машин по origin_url
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
+        "ON cars (origin_url, origin_id)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
+    op.drop_index("ix_images_car_id_fullres", table_name="images")
+    op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,20 +1,26 @@
 x-app-env: &app-env
-  IAAI_DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper}
+  # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
+  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
-  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300}
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600}
  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
-  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-20}
+  # Без лимита beat может забирать слишком большой объём за один запуск.
+  # Консервативный дефолт для anti-fraud: 300 авто за запуск.
+  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-300}
+  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
+  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
+  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
+  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  IAAI_STORAGE_STATE_PATH: ${IAAI_STORAGE_STATE_PATH:-/data/storage_state.json}
-  IAAI_SESSION_MAX_AGE_DAYS: ${IAAI_SESSION_MAX_AGE_DAYS:-30}
-  IAAI_SESSION_SAVE_ON_EXIT: ${IAAI_SESSION_SAVE_ON_EXIT:-true}
-  IAAI_LOGIN_EMAIL: ${IAAI_LOGIN_EMAIL:-}
-  IAAI_LOGIN_PASSWORD: ${IAAI_LOGIN_PASSWORD:-}
+  IAAI_BROWSER_ENGINE: ${IAAI_BROWSER_ENGINE:-auto}
  TZ: ${TZ:-UTC}

 x-env-file: &env-file
@@ -135,9 +141,9 @@ services:
    stop_grace_period: 60s
    command: >
      celery -A iaai_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=1 --pool=prefork
+      --loglevel=info --concurrency=4 --pool=prefork
      --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=20
+      -Q scraping --max-tasks-per-child=5
    healthcheck:
      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
      interval: 60s
--- a/iaai_scraper/api/deps.py
+++ b/iaai_scraper/api/deps.py
@@ -1,4 +1,4 @@
-# Dependency helpers для FastAPI-роутов.
+# Вспомогательные зависимости для FastAPI-роутов.

 from fastapi import Request

--- a/iaai_scraper/api/routes/health.py
+++ b/iaai_scraper/api/routes/health.py
@@ -1,5 +1,7 @@
 # Роут проверки доступности сервиса и соединения с БД.

+import logging
+
 from fastapi import APIRouter, Depends
 from sqlalchemy import text

@@ -7,6 +9,7 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService

 router = APIRouter()
+logger = logging.getLogger("iaai_scraper.api.health")


@router.get("/health")
@@ -18,7 +21,7 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
            session.execute(text("SELECT 1"))
            db_ok = True
    except Exception:
-        pass
+        logger.warning("Health DB check failed", exc_info=True)

    return {
        "status": "ok" if db_ok else "degraded",
--- a/iaai_scraper/browser/factory.py
+++ b/iaai_scraper/browser/factory.py
@@ -4,13 +4,18 @@ import random

 from playwright.sync_api import Browser, BrowserContext, Playwright

+try:
+    from playwright_stealth import stealth_sync
+except ImportError:
+    stealth_sync = None
+
 from ..core.config import Settings

 logger = logging.getLogger("iaai_scraper.browser")


 def _build_init_script() -> str:
-    # JS-патч признаков автоматизации.
+    # Патч признаков автоматизации.
    hardware_concurrency = random.choice([4, 8, 12, 16])
    device_memory = random.choice([4, 8, 16])
    languages = ["en-US", "en"]
@@ -63,17 +68,61 @@ class BrowserFactory:
    def __init__(self, settings: Settings) -> None:
        self.settings = settings

+    def _resolve_engine(self) -> str:
+	    # Выбор движка браузера.
+        engine = self.settings.browser_engine.strip().lower()
+        if engine == "auto":
+            return "firefox" if self.settings.headless else "chromium"
+        if engine in ("firefox", "chromium"):
+            return engine
+        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
+        return "firefox" if self.settings.headless else "chromium"
+
    def create_browser(self, playwright: Playwright) -> Browser:
-        launch_kwargs: dict = {
-            "headless": self.settings.headless,
-            "args": [
+        engine = self._resolve_engine()
+        proxy_dict = self.settings.proxy.to_playwright_dict()
+
+        if engine == "firefox":
+            launch_kwargs: dict = {"headless": self.settings.headless}
+            if proxy_dict:
+                launch_kwargs["proxy"] = proxy_dict
+                logger.info("Using proxy: %s", self.settings.proxy.server)
+            # Параметры Firefox для headless-режима.
+            launch_kwargs["firefox_user_prefs"] = {
+                "dom.webdriver.enabled": False,
+                "useAutomationExtension": False,
+                # Базовые оптимизации для VPS.
+                "media.autoplay.default": 5,
+                "media.volume_scale": "0.0",
+                "media.audio.playback.standalone": False,
+                "dom.ipc.processCount": 1,
+                "dom.ipc.plugins.enabled": False,
+                "browser.cache.disk.enable": False,
+                "browser.cache.memory.enable": True,
+                "browser.cache.memory.max_entry_size": 8192,
+                "network.prefetch-next": False,
+                "network.dns.disablePrefetch": True,
+                "permissions.default.image": 2,
+                "javascript.options.mem.gc_incremental_mark_slice_ms": 20,
+            }
+            logger.info("Launching Firefox (headless=%s)", self.settings.headless)
+            return playwright.firefox.launch(**launch_kwargs)
+
+        # Путь запуска Chromium.
+        args = [
            "--disable-blink-features=AutomationControlled",
            "--no-default-browser-check",
            "--disable-dev-shm-usage",
            "--disable-features=IsolateOrigins,site-per-process",
-            ],
-        }
-        proxy_dict = self.settings.proxy.to_playwright_dict()
+        ]
+        if self.settings.headless:
+            args.append("--headless=new")
+            pw_headless = False
+            logger.info("Using Chromium new-headless mode (--headless=new)")
+        else:
+            pw_headless = False
+
+        launch_kwargs = {"headless": pw_headless, "args": args}
        if proxy_dict:
            launch_kwargs["proxy"] = proxy_dict
            logger.info("Using proxy: %s", self.settings.proxy.server)
@@ -84,34 +133,80 @@ class BrowserFactory:
            logger.warning("Chrome channel launch failed, falling back to Chromium")
            return playwright.chromium.launch(**launch_kwargs)

-    def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext:
+    def create_context(self, browser: Browser) -> BrowserContext:
        viewport = random.choice(self.settings.fingerprint.viewport_presets)
        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
        color_scheme = random.choice(["light", "dark"])

-        context = browser.new_context(
-            storage_state=storage_state or None,
-            user_agent=self.settings.fingerprint.user_agent,
-            viewport=viewport,
-            screen=viewport,
-            device_scale_factor=random.choice([1, 1.25]),
-            is_mobile=False,
-            has_touch=False,
-            locale=self.settings.fingerprint.locale,
-            timezone_id=timezone_id,
-            color_scheme=color_scheme,
-            java_script_enabled=True,
-            ignore_https_errors=False,
-            extra_http_headers={
+        is_firefox = browser.browser_type.name == "firefox"
+
+        ctx_kwargs: dict = {
+            "viewport": viewport,
+            "screen": viewport,
+            "locale": self.settings.fingerprint.locale,
+            "timezone_id": timezone_id,
+            "color_scheme": color_scheme,
+            "java_script_enabled": True,
+            "ignore_https_errors": False,
+        }
+
+        if is_firefox:
+            # Заголовки и user-agent для Firefox.
+            ctx_kwargs["user_agent"] = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
+                "Gecko/20100101 Firefox/128.0"
+            )
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.5",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        else:
+            ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
+            ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
+            ctx_kwargs["is_mobile"] = False
+            ctx_kwargs["has_touch"] = False
+            ctx_kwargs["extra_http_headers"] = {
                "Accept-Language": "en-US,en;q=0.9",
                "DNT": "1",
                "Upgrade-Insecure-Requests": "1",
                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
                "Sec-CH-UA-Mobile": "?0",
                "Sec-CH-UA-Platform": '"Windows"',
-            },
-        )
+            }
+
+        context = browser.new_context(**ctx_kwargs)
        context.set_default_timeout(self.settings.default_timeout_ms)
        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+
+        if not is_firefox:
+            # Патчи маскировки для Chromium.
            context.add_init_script(_build_init_script())
+            if stealth_sync:
+                context.on("page", lambda page: stealth_sync(page))
+                logger.debug("playwright-stealth attached to context")
+
        return context
+
+    @staticmethod
+    def enable_resource_blocking(page) -> None:
+        # Блокируем тяжёлые ресурсы для ускорения загрузки страниц.
+        BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
+        BLOCKED_URL_PATTERNS = (
+            "google-analytics", "googletagmanager", "facebook.net",
+            "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
+            "analytics", "tracking", "adservice",
+        )
+
+        def _handle_route(route):
+            req = route.request
+            if req.resource_type in BLOCKED_TYPES:
+                route.abort()
+                return
+            url = req.url.lower()
+            if any(pat in url for pat in BLOCKED_URL_PATTERNS):
+                route.abort()
+                return
+            route.continue_()
+
+        page.route("**/*", _handle_route)
--- a/iaai_scraper/browser/listing.py
+++ b/iaai_scraper/browser/listing.py
@@ -1,5 +1,6 @@
 import logging
 import re
+import time
 from dataclasses import asdict, dataclass, field
 from typing import Any
 from urllib.parse import urljoin
@@ -11,7 +12,7 @@ from ..core.config import Settings
 from ..core.utils import first_non_empty

 logger = logging.getLogger("iaai_scraper.listing")
-VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)


@dataclass(slots=True)
@@ -37,15 +38,35 @@ class ListingCollector:

 	def open_cars_listing(self, page: Page) -> None:
 		logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
-		page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded")
+		url = self.settings.listing.cars_url
+		last_err = None
+		for attempt in range(3):
 			try:
-			page.wait_for_load_state("networkidle", timeout=15000)
-		except Exception:
-			logger.debug("networkidle timeout on listing page, continuing with current state")
+				page.goto(url, wait_until="commit", timeout=60_000)
+				last_err = None
+				break
+			except Exception as e:
+				last_err = e
+				logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
+				# Небольшой backoff при ошибках открытия листинга.
+				time.sleep(5 * (attempt + 1))
+		if last_err:
+			logger.warning("All goto attempts failed, trying JS navigation")
 			try:
-			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000)
+				page.evaluate(f"window.location.href = '{url}'")
 			except Exception:
-			logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully")
+				pass
+		# Быстрая проверка готовности страницы.
+		try:
+			page.wait_for_load_state("domcontentloaded", timeout=12_000)
+		except Exception:
+			pass
+		try:
+			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=4_000)
+		except Exception:
+			# Короткая пауза вместо длинного sleep.
+			time.sleep(0.25)
+		logger.info("Listing page URL: %s", page.url)
 		self.pacer.after_listing_open()

 	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
@@ -59,28 +80,48 @@ class ListingCollector:
 		return applied

 	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
-		anchors = page.locator("a[href*='/VehicleDetail/']")
-		total = min(anchors.count(), self.settings.listing.page_link_limit)
+		# Считываем ссылки одним проходом по DOM.
+		raw_items = page.eval_on_selector_all(
+			"a[href*='/VehicleDetail/']",
+			"""
+			(nodes) => nodes.map((a) => ({
+				href: a.getAttribute('href') || '',
+				title: a.getAttribute('title') || '',
+				text: (a.textContent || '').trim(),
+			}))
+			""",
+		)
+		total = min(len(raw_items), self.settings.listing.page_link_limit)
 		links: list[ListingVehicleLink] = []
 		seen: set[str] = set()
 		for idx in range(total):
-			anchor = anchors.nth(idx)
-			href = anchor.get_attribute("href") or ""
+			item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
+			href = str(item.get("href") or "")
 			match = VEHICLE_HREF_RE.search(href)
 			if not match:
 				continue
+			lot_number = match.group(1)
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
 			seen.add(absolute)
-			title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or ""
-			links.append(ListingVehicleLink(href=absolute, title=str(title).strip()))
+			title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
+			links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
 			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
 		next_page_detected = self._has_next_page(page)
 		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)

 	def go_to_next_page(self, page: Page) -> bool:
+		# Запоминаем первую ссылку текущей страницы для определения смены контента.
+		old_first_href = ""
+		try:
+			first_link = page.locator("a[href*='/VehicleDetail/']").first
+			if first_link.count() > 0:
+				old_first_href = first_link.get_attribute("href") or ""
+		except Exception:
+			pass
+
 		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
 		for selector in selectors:
 			locator = page.locator(selector).first
@@ -93,32 +134,118 @@ class ListingCollector:
 				continue
 			self.pacer.move_mouse_to(page, locator)
 			locator.click()
+
+			# Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться.
+			if old_first_href:
 				try:
-				page.wait_for_load_state("networkidle", timeout=15000)
+					page.wait_for_function(
+						f"""() => {{
+							const a = document.querySelector("a[href*='/VehicleDetail/']");
+							return a && a.getAttribute('href') !== '{old_first_href}';
+						}}""",
+						timeout=8000,
+					)
+				except Exception:
+					pass
+			else:
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=15000)
+				except Exception:
+					pass
+
+			try:
+				page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
 			except Exception:
 				pass
 			self.pacer.after_page_change()
 			return True
 		return False

-	def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]:
+	def collect_listing_links(
+		self,
+		page: Page,
+		*,
+		make: str | None = None,
+		model: str | None = None,
+		known_origin_ids: set[str] | None = None,
+	) -> dict[str, Any]:
 		self.open_cars_listing(page)
 		applied_filters = self.apply_filters(page, make=make, model=model)
 		pages: list[dict[str, object]] = []
 		all_links: list[str] = []
+		early_stopped = False
+		threshold = self.settings.listing.early_stop_threshold
+
 		for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
 			page_result = self.collect_current_page(page, page_number=page_number)
-			pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected})
+			pages.append({
+				"page_number": page_result.page_number,
+				"source_url": page_result.source_url,
+				"links_found": len(page_result.vehicle_links),
+				"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
+				"next_page_detected": page_result.next_page_detected,
+			})
 			for item in page_result.vehicle_links:
 				if item.href not in all_links:
 					all_links.append(item.href)
 				if len(all_links) >= self.settings.listing.max_vehicles_per_run:
 					break
-			if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected:
+
+			# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
+			# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
+			if (
+				known_origin_ids is not None
+				and threshold > 0.0
+				and page_result.vehicle_links
+			):
+				page_known = sum(
+					1 for item in page_result.vehicle_links
+					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
+				)
+				page_new = len(page_result.vehicle_links) - page_known
+				ratio = page_known / len(page_result.vehicle_links)
+				# Останавливаемся только если нет новых И порог превышен
+				if ratio >= threshold and page_new == 0:
+					logger.info(
+						"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
+						page_number, ratio * 100, page_known,
+						len(page_result.vehicle_links), threshold * 100,
+					)
+					early_stopped = True
+					break
+				elif page_new > 0 and ratio >= threshold:
+					logger.info(
+						"Page %d: %.0f%% known but %d new found — продолжаем",
+						page_number, ratio * 100, page_new,
+					)
+
+			if (
+				len(all_links) >= self.settings.listing.max_vehicles_per_run
+				or self.settings.listing.collect_current_page_only
+				or not self.settings.listing.include_pagination
+				or not page_result.next_page_detected
+			):
 				break
 			if not self.go_to_next_page(page):
 				break
-		return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}}
+
+		return {
+			"listing_url": self.settings.listing.cars_url,
+			"applied_filters": applied_filters,
+			"pages_collected": len(pages),
+			"vehicles_collected": len(all_links),
+			"vehicle_urls": all_links,
+			"early_stopped": early_stopped,
+			"pages": pages,
+			"strategy": {
+				"sequential": True,
+				"collect_current_page_only": self.settings.listing.collect_current_page_only,
+				"include_pagination": self.settings.listing.include_pagination,
+				"max_pages_per_run": self.settings.listing.max_pages_per_run,
+				"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
+				"early_stop_threshold": threshold,
+			},
+		}

 	@staticmethod
 	def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
@@ -131,7 +258,11 @@ class ListingCollector:
 				locator.fill(value)
 				page.keyboard.press("Enter")
 				try:
-					page.wait_for_load_state("networkidle", timeout=15000)
+					page.wait_for_load_state("domcontentloaded", timeout=15000)
+				except Exception:
+					pass
+				try:
+					page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
 				except Exception:
 					pass
 				return True
--- a/iaai_scraper/browser/network.py
+++ b/iaai_scraper/browser/network.py
@@ -13,7 +13,7 @@ logger = logging.getLogger("iaai_scraper.network")

@dataclass
 class NetworkCapture:
-    # XHR/fetch перехватчик.
+    # Перехватчик сетевых запросов.

    settings: Settings
    requests: list[dict[str, Any]] = field(default_factory=list)
@@ -21,13 +21,21 @@ class NetworkCapture:
    _seen_req: set[str] = field(default_factory=set)
    _seen_resp: set[str] = field(default_factory=set)
    _origin: str | None = None
+    _page: Any = field(default=None)

    def attach(self, page: Page, origin_url: str | None = None) -> None:
+        # Снимаем старые подписки перед повторным attach.
+        try:
+            page.remove_listener("request", self._on_request)
+            page.remove_listener("response", self._on_response)
+        except Exception:
+            pass
        # Подписка на сетевые события
        try:
            self._origin = urlparse(origin_url or page.url).netloc.lower() or None
        except Exception:
            self._origin = None
+        self._page = page
        page.on("request", self._on_request)
        page.on("response", self._on_response)

--- a/iaai_scraper/browser/pace.py
+++ b/iaai_scraper/browser/pace.py
@@ -7,7 +7,7 @@ from ..core.config import Settings


 class HumanPacer:
-    # Random паузы между действиями.
+    # Случайные паузы между действиями.

    def __init__(self, settings: Settings) -> None:
        self.settings = settings
--- a/iaai_scraper/core/config.py
+++ b/iaai_scraper/core/config.py
@@ -84,18 +84,18 @@ class CaptureConfig:
@dataclass(slots=True)
 class HumanPaceConfig:
    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
-    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 2.5)
-    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 4.5)
-    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 2.0)
-    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 4.0)
-    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.5)
-    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 1.5)
-    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.3)
-    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 1.0)
-    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.5)
-    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 1.5)
-    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 3.0)
-    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 6.0)
+    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
+    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
+    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
+    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
+    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
+    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
+    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
+    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
+    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
+    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
+    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
+    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)


 #  Конфиг сбора листинга (URL, лимиты страниц и машин) 
@@ -103,11 +103,14 @@ class HumanPaceConfig:
@dataclass(slots=True)
 class ListingConfig:
    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
-    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 5)
-    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 100)
-    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 200)
+    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
+    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
+    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)


 # - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
@@ -135,13 +138,16 @@ class RedisConfig:
 class CeleryConfig:
    broker_url: str = _env_str("CELERY_BROKER_URL", "")
    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
-    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 600)
-    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 900)
-    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 1)
-    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 20)
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
-    beat_sync_limit: int = _env_int("CELERY_BEAT_SYNC_LIMIT", 26)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
+    block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)


 # --- Конфиг прокси (server, username, password) ---
@@ -167,38 +173,22 @@ class ProxyConfig:
        return result


-# --- Конфиг сессии браузера (куки, storage_state) ---
-
-@dataclass(slots=True)
-class SessionConfig:
-    storage_state_path: str | None = _env_optional_str("IAAI_STORAGE_STATE_PATH")
-    max_age_days: int = _env_int("IAAI_SESSION_MAX_AGE_DAYS", 30)
-    save_on_exit: bool = _env_bool("IAAI_SESSION_SAVE_ON_EXIT", True)
-    login_email: str | None = _env_optional_str("IAAI_LOGIN_EMAIL")
-    login_password: str | None = _env_optional_str("IAAI_LOGIN_PASSWORD")
-    login_url: str = _env_str("IAAI_LOGIN_URL", "https://www.iaai.com/Login")
-
-    @property
-    def enabled(self) -> bool:
-        return bool(self.storage_state_path)
-
-    @property
-    def has_credentials(self) -> bool:
-        return bool(self.login_email and self.login_password)
-
-
 # --- Главный объект настроек: собирает все блоки конфигурации ---

@dataclass(slots=True)
 class Settings:
    home_url: str = "https://www.iaai.com/"
    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
-    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 800)
+    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
+    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
+    fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
+    fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
    headless: bool = _env_bool("IAAI_HEADLESS", True)
+    browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
@@ -215,7 +205,14 @@ class Settings:
    redis: RedisConfig = field(default_factory=RedisConfig)
    celery: CeleryConfig = field(default_factory=CeleryConfig)
    proxy: ProxyConfig = field(default_factory=ProxyConfig)
-    session: SessionConfig = field(default_factory=SessionConfig)
+
+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs
+
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources

 # Глобальный синглтон — используется по умолчанию во всех модулях.
 settings = Settings()
--- a/iaai_scraper/core/exceptions.py
+++ b/iaai_scraper/core/exceptions.py
@@ -1,10 +1,10 @@
 class ScraperError(Exception):
-    """Base scraper exception."""
+    """Базовое исключение скрапера."""


 class AntiBotDetectedError(ScraperError):
-    """Raised when the target website appears to block automation."""
+    """Вызывается, когда сайт блокирует автоматизацию."""


 class SiteStructureChangedError(ScraperError):
-    """Raised when the page shape changed and required data is missing."""
+    """Вызывается, когда структура страницы изменилась и данных не хватает."""
--- a/iaai_scraper/core/logs.py
+++ b/iaai_scraper/core/logs.py
@@ -2,7 +2,7 @@ import logging
 import sys
 from contextvars import ContextVar

-# ContextVar хранит trace_id текущего потока/корутины.
+# Храним trace_id текущего потока/корутины.
 TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")


--- a/iaai_scraper/core/utils.py
+++ b/iaai_scraper/core/utils.py
@@ -1,7 +1,7 @@
 import json
 import re
 from pathlib import Path
-from typing import Any, Iterable
+from typing import Any, Callable, Iterable


 def save_to_json(data: Any, filename: str | Path) -> None:
@@ -17,7 +17,7 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
    return None


-# Regex для VIN, lot, price.
+# Регулярные выражения для VIN, lot и price.
 VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
 LOT_RE = re.compile(r"\b(\d{7,10})\b")
 PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
@@ -37,3 +37,36 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
        for item in obj:
            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
    return found
+
+
+def deep_find_all_keys(
+    payloads: list,
+    field_map: dict[str, set[str]],
+    max_depth: int = 64,
+) -> dict[str, list]:
+    """Извлекает все нужные поля за один проход по JSON."""
+    # Готовим обратную карту: нормализованный ключ -> имя поля.
+    reverse: dict[str, str] = {}
+    for field_name, keys in field_map.items():
+        for k in keys:
+            reverse[k.lower()] = field_name
+
+    result: dict[str, list] = {f: [] for f in field_map}
+
+    def _recurse(obj: Any, depth: int) -> None:
+        if depth >= max_depth:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                field = reverse.get(k.lower())
+                if field is not None:
+                    result[field].append(v)
+                _recurse(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _recurse(item, depth + 1)
+
+    for payload in payloads:
+        _recurse(payload, 0)
+
+    return result
--- a/iaai_scraper/parsing/mapper.py
+++ b/iaai_scraper/parsing/mapper.py
@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord


 class CarMapper:
-	# IAAI data → CarRecord.
+	# Преобразование данных IAAI в CarRecord.

 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -72,12 +72,18 @@ class CarMapper:
 			],
 			self._to_money_int,
 		)
-		mileage = self._first_parsed_int(
-			[core.get("odometer"), vehicle_summary.get("odometer"), 0],
-			self._to_int,
-		) or 0
+		mileage = self._parse_odometer(
+			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
+		)
 		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
 		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+		# Пытаемся определить привод из строки двигателя.
+		if not drive or drive == "NA":
+			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
+			if engine_text:
+				inferred_drive = self._normalize_drive(engine_text)
+				if inferred_drive and inferred_drive != "NA":
+					drive = inferred_drive
 		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
 		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
 		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
@@ -107,7 +113,7 @@ class CarMapper:
 				]
 			)
 		)
-		slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id])))
+		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
 		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
 		origin = "IAAI"

@@ -206,6 +212,25 @@ class CarMapper:
 			return parsed
 		return None

+	@staticmethod
+	def _parse_odometer(value: Any) -> int:
+		# Разбор пробега из строк IAAI.
+		if value is None:
+			return 0
+		text = str(value).strip()
+		if not text:
+			return 0
+		lowered = text.lower()
+		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
+			return 0
+		# Извлекаем число из строкового формата одометра.
+		numbers = re.findall(r"[\d,]+", text)
+		for num_str in numbers:
+			clean = num_str.replace(",", "")
+			if clean.isdigit() and int(clean) > 0:
+				return int(clean)
+		return 0
+
 	def _to_engine_cc(self, value: Any) -> int | None:
 		# Поддержка литров и cc.
 		text = str(value).lower().strip() if value is not None else ""
@@ -344,13 +369,13 @@ class CarMapper:
 				return preview
 		return url

-	# ---------- parser_id ----------
+	# Формирование parser_id.

-	_PARSER_ID_ALPHABET = ascii_letters + digits  # a-zA-Z0-9
+	_PARSER_ID_ALPHABET = ascii_letters + digits

 	@classmethod
 	def _generate_parser_id(cls, origin_id: str) -> str:
-		"""Deterministic car-XXXX...  (prefix 'car-' + 22 alphanumeric chars)."""
+		# Стабильный parser_id по origin_id.
 		digest = hashlib.sha256(origin_id.encode()).digest()
 		alphabet = cls._PARSER_ID_ALPHABET
 		base = len(alphabet)
@@ -363,7 +388,7 @@ class CarMapper:

 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
 		# Формат: iaai:{lot_number} (аналог copart:94323985).
-		for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]:
+		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
 			text = self._as_str(value)
 			if text:
 				return f"iaai:{text}"
--- a/iaai_scraper/parsing/parser.py
+++ b/iaai_scraper/parsing/parser.py
@@ -4,16 +4,22 @@ import logging
 import re
 from typing import Any

-from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty
+from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty

 logger = logging.getLogger("iaai_scraper.parsers")


 class VehicleParser:
-	# DOM + JSON парсер страницы авто.
+	# Парсер данных страницы автомобиля.
+
+	# Регулярные выражения для парсинга и детекции защиты.
+	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
+	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
+	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
+	_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
+	_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)

 	SUMMARY_KEY_MAP = {
-		"vin": {"vin", "vehicleidentificationnumber"},
 		"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
 		"year": {"year"},
 		"make": {"make", "manufacturer", "brand"},
@@ -32,10 +38,10 @@ class VehicleParser:
 		"seller": {"seller", "sellername"},
 		"location": {"location", "branchname", "auctionlocation", "branch"},
 		"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
-		"body_type": {"bodytype", "bodystyle"},
-		"drive": {"driveline", "drive"},
-		"gearbox": {"transmission", "gearbox"},
-		"engine": {"engine", "enginevolume"},
+		"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
+		"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
+		"gearbox": {"transmission", "gearbox", "transmissiontype"},
+		"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
 		"fuel_type": {"fueltype", "fuel"},
 		"cylinders": {"cylinders", "cylindercount"},
 		"color": {"color", "primarycolor", "exteriorcolor"},
@@ -43,24 +49,35 @@ class VehicleParser:

 	DOM_LABEL_MAP: dict[str, str] = {
 		"stock #": "lot_number",
-		"vin (status)": "vin",
-		"vin": "vin",
+		"stock": "lot_number",
 		"primary damage": "primary_damage",
 		"secondary damage": "secondary_damage",
 		"odometer": "odometer",
+		"odometer (miles)": "odometer",
+		"mileage": "odometer",
 		"body style": "body_type",
+		"body type": "body_type",
+		"vehicle type": "body_type",
 		"engine": "engine",
+		"engine type": "engine",
 		"transmission": "gearbox",
 		"drive line type": "drive",
+		"driveline type": "drive",
+		"drive line": "drive",
+		"driveline": "drive",
+		"drive type": "drive",
 		"fuel type": "fuel_type",
+		"fuel": "fuel_type",
 		"cylinders": "cylinders",
 		"exterior/interior": "color",
 		"exterior color": "color",
+		"color": "color",
 		"model": "model",
 		"series": "trim",
 		"selling branch": "location",
 		"vehicle location": "vehicle_location",
 		"auction date and time": "auction_date",
+		"sale date": "auction_date",
 		"lane/run #": "lane",
 		"actual cash value": "actual_cash_value",
 		"estimated repair cost": "estimated_repair_cost",
@@ -69,6 +86,7 @@ class VehicleParser:
 		"title/sale doc brand": "title_brand",
 		"start code": "run_and_drive",
 		"key": "keys",
+		"keys": "keys",
 		"manufactured in": "manufactured_in",
 		"vehicle class": "vehicle_class",
 	}
@@ -79,15 +97,41 @@ class VehicleParser:
 			return result
 		lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
 		known_labels = set(self.DOM_LABEL_MAP.keys())
+		skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
+		max_fields = len(set(self.DOM_LABEL_MAP.values()))
+
 		for i, line in enumerate(lines):
-			clean = line.rstrip(":").lower().strip()
-			if clean in known_labels and i + 1 < len(lines):
+			# Ранний выход, когда уже нашли все поля.
+			if len(result) >= max_fields:
+				break
+
+			# Сценарий 1: "метка: значение" в одной строке.
+			colon_pos = line.find(":")
+			if colon_pos > 0:
+				label_part = line[:colon_pos].strip().lower()
+				value_part = line[colon_pos + 1:].strip()
+				if label_part in known_labels and value_part and value_part.lower() not in skip_values:
+					field_name = self.DOM_LABEL_MAP[label_part]
+					if field_name not in result or not result[field_name]:
+						result[field_name] = value_part
+						continue
+
+			# Сценарий 2: метка и значение на соседних строках.
+			clean = line.rstrip(":").strip().lower()
+			clean_alt = clean.rstrip("#").strip()
+			matched_label = None
+			if clean in known_labels:
+				matched_label = clean
+			elif clean_alt in known_labels:
+				matched_label = clean_alt
+
+			if matched_label and i + 1 < len(lines):
 				value = lines[i + 1].strip()
 				if value.rstrip(":").lower().strip() in known_labels:
 					continue
-				if value.lower() in ("more actions", "view", "print", "share", "back to results"):
+				if value.lower() in skip_values:
 					continue
-				field_name = self.DOM_LABEL_MAP[clean]
+				field_name = self.DOM_LABEL_MAP[matched_label]
 				if field_name not in result or not result[field_name]:
 					result[field_name] = value
 		return result
@@ -120,11 +164,11 @@ class VehicleParser:
 			page_title = title_match.group(1).strip()
 		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)

+		# Один проход по payload для всех полей.
+		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
+
 		summary: dict[str, Any] = {"source_url": vehicle_url}
-		for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
-			values: list[Any] = []
-			for payload in payloads:
-				values.extend(deep_find_key(payload, candidate_keys))
+		for field, values in all_found.items():
 			if field in dom_kv:
 				values.append(dom_kv[field])
 			summary[field] = first_non_empty(values)
@@ -133,11 +177,6 @@ class VehicleParser:
 		summary["make"] = summary.get("make") or title_parsed.get("make")
 		summary["model"] = summary.get("model") or title_parsed.get("model")
 		summary["trim"] = summary.get("trim") or dom_kv.get("trim")
-		summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html)
-		if summary.get("vin") and isinstance(summary["vin"], str):
-			vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip()
-			vin_match = VIN_RE.search(vin_clean)
-			summary["vin"] = vin_match.group(1) if vin_match else vin_clean
 		summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
 		summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
 		for dom_field, dom_value in dom_kv.items():
@@ -147,7 +186,7 @@ class VehicleParser:
 		if not summary.get("actual_cash_value") and prices:
 			summary["actual_cash_value"] = prices[0]
 		if not summary.get("buy_now"):
-			buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE)
+			buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
 			if buy_now_match:
 				summary["buy_now"] = buy_now_match.group(1)
 			elif prices:
@@ -160,25 +199,30 @@ class VehicleParser:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
 				payloads.append(p)
-				for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
+				# Дополнительный проход по встроенному JSON.
+				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
+				for field, vals in extra.items():
 					if not summary.get(field):
-						val = first_non_empty(deep_find_key(p, candidate_keys))
-						if val:
-							summary[field] = val
+						v = first_non_empty(vals)
+						if v:
+							summary[field] = v

+		# Передаём image_urls без повторного извлечения.
+		image_urls = summary.get("image_urls") or []
 		return {
 			"vehicle_summary": summary,
-			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url),
+			"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
 			"embedded_json": embedded,
 			"dom_hints": self._dom_hints(dom_text),
 			"access_notes": self._build_access_notes(summary, responses),
 		}

-	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
+	def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
+		if image_urls is None:
 			image_urls = self._extract_image_urls(payloads, "", vehicle_url)
 		return {
 			"vehicle_core": {
-				"vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"),
+				"lot_number": summary.get("lot_number"), "year": summary.get("year"),
 				"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
 				"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
 				"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
@@ -238,7 +282,6 @@ class VehicleParser:
 		endpoints = [item.get("url", "") for item in responses]
 		dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
 		return {
-			"vin_visible": bool(summary.get("vin")),
 			"images_visible": bool(summary.get("image_urls")),
 			"network_json_count": len(responses),
 			"possible_captcha": bool(dom_hints.get("has_captcha_text")),
@@ -266,11 +309,6 @@ class VehicleParser:
 				return "JPY"
 		return "USD"

-	@staticmethod
-	def _extract_vin(text: str) -> str | None:
-		match = VIN_RE.search(text or "")
-		return match.group(1) if match else None
-
 	@staticmethod
 	def _extract_lot_number(text: str) -> str | None:
 		match = LOT_RE.search(text or "")
@@ -287,6 +325,9 @@ class VehicleParser:
 		for script_text in scripts:
 			if "{" not in script_text and "[" not in script_text:
 				continue
+			# Пропускаем слишком большие минифицированные блоки.
+			if len(script_text) > 51_200:
+				continue
 			try:
 				parsed = json.loads(script_text.strip())
 			except Exception:
@@ -361,8 +402,7 @@ class VehicleParser:
 			"has_buy_now_text": "buy now" in lowered,
 			"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
 			"has_damage_text": "damage" in lowered,
-			"has_vin_text": "vin" in lowered,
 			"has_title_text": "title" in lowered,
-			"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
-			"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
+			"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
+			"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
 		}
--- a/iaai_scraper/scraper.py
+++ b/iaai_scraper/scraper.py
--- a/iaai_scraper/storage/db.py
+++ b/iaai_scraper/storage/db.py
@@ -1,9 +1,10 @@
 import logging
 from contextlib import contextmanager
 from datetime import datetime, timezone
-from typing import Iterator
+from typing import Any, Iterator

-from sqlalchemy import create_engine, or_, select
+from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session, sessionmaker

 from ..core.config import Settings
@@ -18,6 +19,8 @@ CAR_DB_FIELDS = {
 	if col.key not in ("id",)
 }

+_IN_CHUNK_SIZE = 5000
+

 class PersistenceService:

@@ -99,21 +102,250 @@ class PersistenceService:
 			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
 			return {str(row[0]) for row in rows if row and row[0]}

+	def get_existing_urls_and_ids(
+		self, origin_urls: list[str], origin_ids: list[str],
+	) -> tuple[set[str], set[str]]:
+		# Загрузка существующих URL и origin_id.
+		if not origin_urls and not origin_ids:
+			return set(), set()
+		urls: set[str] = set()
+		ids: set[str] = set()
+		with self.session_scope() as session:
+			max_len = max(len(origin_urls), len(origin_ids), 1)
+			for i in range(0, max_len, _IN_CHUNK_SIZE):
+				url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+				id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+				conditions = []
+				if url_chunk:
+					conditions.append(Car.origin_url.in_(url_chunk))
+				if id_chunk:
+					conditions.append(Car.origin_id.in_(id_chunk))
+				if not conditions:
+					continue
+				rows = session.execute(
+					select(Car.origin_url, Car.origin_id).where(or_(*conditions))
+				).all()
+				for r in rows:
+					if r[0]:
+						urls.add(str(r[0]))
+					if r[1]:
+						ids.add(str(r[1]))
+		return urls, ids
+
 	@staticmethod
 	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
-		for image_payload in images:
-			session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id))
+		if not images:
+			return
+		session.add_all([
+			Image(
+				fullres_image=str(img["fullres_image"]),
+				preview_image=str(img["preview_image"]),
+				order_index=int(img.get("order_index", 0)),
+				car_id=car_id,
+			)
+			for img in images
+		])

 	@staticmethod
 	def _car_payload(record: CarRecord) -> dict[str, object]:
 		payload = record.model_dump(mode="python")
 		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}

+	def _is_postgres(self) -> bool:
+		return self.engine.dialect.name == "postgresql"
+
+	def _load_existing_cars(
+		self,
+		session: Session,
+		origin_ids: list[str],
+		origin_urls: list[str],
+	) -> tuple[dict[str, Car], dict[str, Car]]:
+		existing_by_id: dict[str, Car] = {}
+		existing_by_url: dict[str, Car] = {}
+		if not origin_ids and not origin_urls:
+			return existing_by_id, existing_by_url
+
+		existing_cars: list[Car] = []
+		max_len = max(len(origin_ids), len(origin_urls), 1)
+		for i in range(0, max_len, _IN_CHUNK_SIZE):
+			id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+			url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+			conditions = []
+			if id_chunk:
+				conditions.append(Car.origin_id.in_(id_chunk))
+			if url_chunk:
+				conditions.append(Car.origin_url.in_(url_chunk))
+			if not conditions:
+				continue
+			rows = session.execute(
+				select(Car).where(or_(*conditions))
+			).scalars().all()
+			existing_cars.extend(rows)
+
+		for car in existing_cars:
+			if car.origin_id:
+				existing_by_id[car.origin_id] = car
+			if car.origin_url:
+				existing_by_url[car.origin_url] = car
+		return existing_by_id, existing_by_url
+
+	def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
+		existing_images_map: dict[int, set[str]] = {}
+		if not car_ids:
+			return existing_images_map
+
+		car_id_list = list(car_ids)
+		for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
+			chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
+			img_rows = session.execute(
+				select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
+			).all()
+			for cid, furl in img_rows:
+				existing_images_map.setdefault(int(cid), set()).add(str(furl))
+		return existing_images_map
+
+	@staticmethod
+	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
+		return {
+			key: getattr(insert_stmt.excluded, key)
+			for key in CAR_DB_FIELDS
+		}
+
+	def _replace_images_for_car(
+		self,
+		session: Session,
+		car_id: int,
+		images: list[dict[str, object]],
+		origin_id: str,
+	) -> int:
+		nested = session.begin_nested()
+		try:
+			session.execute(delete(Image).where(Image.car_id == car_id))
+			self._add_images(session, car_id, images)
+			session.flush()
+			nested.commit()
+			return len(images)
+		except Exception:
+			nested.rollback()
+			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
+			return 0
+
+	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			entries: list[dict[str, object]] = []
+			upsert_payloads: list[dict[str, object]] = []
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+				car_by_id = existing_by_id.get(record.origin_id)
+				car_by_url = existing_by_url.get(record.origin_url)
+				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					entry["car_id"] = int(car_by_url.id)
+					updated += 1
+				else:
+					upsert_payloads.append(payload)
+					if car_by_id is not None:
+						updated += 1
+					else:
+						inserted += 1
+				entries.append(entry)
+
+			session.flush()
+
+			if upsert_payloads:
+				insert_stmt = pg_insert(Car).values(upsert_payloads)
+				upsert_stmt = insert_stmt.on_conflict_do_update(
+					index_elements=[Car.origin_id],
+					set_=self._postgres_upsert_set_map(insert_stmt),
+				).returning(Car.id, Car.origin_id)
+				rows = session.execute(upsert_stmt).all()
+				car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
+				for entry in entries:
+					if entry["car_id"] is not None:
+						continue
+					record = entry["record"]
+					car_id = car_ids_by_origin_id.get(record.origin_id)
+					if car_id is None:
+						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
+					entry["car_id"] = car_id
+
+			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
+			existing_images_map = self._load_existing_image_urls(session, car_ids)
+			images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			replace_ids: list[int] = []
+
+			for entry in entries:
+				car_id = int(entry["car_id"])
+				images = entry["images"]
+				new_image_urls = {
+					str(img.get("fullres_image", ""))
+					for img in images
+					if img.get("fullres_image")
+				}
+				old_image_urls = existing_images_map.get(car_id, set())
+				if new_image_urls != old_image_urls:
+					replace_ids.append(car_id)
+					images_by_car_id[car_id] = images
+				else:
+					images_total += len(old_image_urls)
+
+			if replace_ids:
+				for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
+					chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car_id in replace_ids:
+					images = images_by_car_id[car_id]
+					self._add_images(session, car_id, images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
 	def upsert_car(self, record: CarRecord):
-		# Insert/update автомобиля по origin_id/origin_url.
+		# Вставка или обновление автомобиля по origin_id/origin_url.
 		payload = self._car_payload(record)
 		images = [image.model_dump(mode="python") for image in record.images]
 		with self.session_scope() as session:
+			if self._is_postgres():
+				car_by_url = session.execute(
+					select(Car).where(Car.origin_url == record.origin_url)
+				).scalar_one_or_none()
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					session.flush()
+					car_id = int(car_by_url.id)
+					action = "updated"
+				else:
+					existed = session.execute(
+						select(Car.id).where(Car.origin_id == record.origin_id)
+					).scalar_one_or_none() is not None
+					insert_stmt = pg_insert(Car).values(**payload)
+					upsert_stmt = insert_stmt.on_conflict_do_update(
+						index_elements=[Car.origin_id],
+						set_=self._postgres_upsert_set_map(insert_stmt),
+					).returning(Car.id)
+					car_id = int(session.execute(upsert_stmt).scalar_one())
+					action = "updated" if existed or car_by_url is not None else "inserted"
+
+				images_upserted = self._replace_images_for_car(
+					session, car_id, images, record.origin_id,
+				)
+				return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
+
 			car = session.execute(
 				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
 			).scalar_one_or_none()
@@ -128,19 +360,217 @@ class PersistenceService:
 					setattr(car, key, value)
 				car.last_seen_at = record.last_seen_at
 				session.flush()
-				nested = session.begin_nested()
+				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
+				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
+			self._add_images(session, int(car.id), images)
+			session.flush()
+			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Пакетный upsert нескольких автомобилей в одной транзакции.
+
+		Оптимизации:
+		- Дедупликация записей по origin_id перед вставкой.
+		- Chunked IN-queries для больших списков (обход лимита PG параметров).
+		- Пропуск перезаписи изображений, если набор URL не изменился.
+		- Один DELETE по car_id IN (...) вместо удаления по одному.
+		- Fallback на по-одному upsert если batch commit упал.
+		"""
+		# ── Дедупликация записей внутри батча ──
+		seen_ids: dict[str, int] = {}
+		unique_records: list[CarRecord] = []
+		for idx, r in enumerate(records):
+			key = r.origin_id or r.origin_url
+			if key in seen_ids:
+				logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
+				continue
+			seen_ids[key] = idx
+			unique_records.append(r)
+
+		if len(unique_records) < len(records):
+			logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
+		records = unique_records
+
 		try:
-					for image in list(car.images):
-						session.delete(image)
+			return self._upsert_cars_batch_inner(records)
+		except Exception as exc:
+			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
+			return self._upsert_cars_individually(records)
+
+	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Внутренняя реализация batched upsert (одна транзакция)."""
+		if self._is_postgres():
+			return self._upsert_cars_batch_postgres(records)
+
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			# Получаем существующие записи chunked-запросами.
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
+			existing_car_ids = set()
+			for record in records:
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is not None:
+					existing_car_ids.add(int(car.id))
+
+			# Строим маппинг car_id → set(image_urls) для сравнения.
+			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
+
+			new_cars: list[tuple[Car, list[dict]]] = []
+			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is None:
+					car = Car(**payload)
+					session.add(car)
+					inserted += 1
+					new_cars.append((car, images))
+				else:
+					for key, value in payload.items():
+						setattr(car, key, value)
+					car.last_seen_at = record.last_seen_at
+					updated += 1
+
+					# Проверяем, изменились ли изображения.
+					new_image_urls = {img.get("fullres_image", "") for img in images}
+					old_image_urls = existing_images_map.get(int(car.id), set())
+					if new_image_urls != old_image_urls:
+						update_cars_needing_images.append((car, images))
+					else:
+						images_total += len(old_image_urls)
+
+			# Один flush для всех вставок.
 			session.flush()
+
+			# Добавляем изображения для новых автомобилей.
+			for car, images in new_cars:
 				self._add_images(session, int(car.id), images)
-					session.flush()
-					nested.commit()
-				except Exception:
-					nested.rollback()
-					logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id)
-					images = []
-				return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+				images_total += len(images)
+
+			# Массово обновляем изображения только для машин с изменёнными картинками.
+			if update_cars_needing_images:
+				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
+				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
+					chunk = update_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car, images in update_cars_needing_images:
 					self._add_images(session, int(car.id), images)
-			session.flush()
-			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
+		# Fallback на поштучный upsert.
+		inserted = 0
+		updated = 0
+		images_total = 0
+		for record in records:
+			try:
+				result = self.upsert_car(record)
+				action = result.get("action", "inserted")
+				if action == "inserted":
+					inserted += 1
+				else:
+					updated += 1
+				images_total += int(result.get("images_upserted", 0))
+			except Exception as exc:
+				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
+		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
+		if not active_origin_ids:
+			return 0
+		with self.session_scope() as session:
+			stmt = (
+				update(Car)
+				.where(Car.origin_id.notin_(active_origin_ids))
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like("iaai:%"))
+				.values(is_sold=True)
+			)
+			result = session.execute(stmt)
+			count = result.rowcount or 0
+			if count:
+				logger.info("Marked %d cars as sold (no longer in listing)", count)
+			return count
+
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
+		"""Помечает авто как проданные, если их URL нет в активном листинге.
+
+		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
+		что кардинально быстрее при больших объёмах (100K+ URLs).
+		"""
+		if not active_origin_urls:
+			return 0
+
+		is_postgres = "postgresql" in self.settings.database.url
+
+		with self.session_scope() as session:
+			if is_postgres:
+				# Создаём временную таблицу с активными URL.
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
+				session.execute(text("TRUNCATE _active_urls"))
+
+				# Вставляем активные URL чанками.
+				url_list = list(active_origin_urls)
+				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
+					chunk = url_list[i:i + _IN_CHUNK_SIZE]
+					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
+					params = {f"u{j}": url for j, url in enumerate(chunk)}
+					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
+
+				# Создаём индекс на временной таблице для ускорения JOIN.
+				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
+
+				# Массовая пометка проданных в PostgreSQL.
+				result = session.execute(text("""
+					UPDATE cars
+					SET is_sold = TRUE
+					FROM (
+						SELECT c.id
+						FROM cars c
+						LEFT JOIN _active_urls a ON c.origin_url = a.url
+						WHERE a.url IS NULL
+						AND c.is_sold = FALSE
+						AND c.origin_id LIKE 'iaai:%%'
+					) sub
+					WHERE cars.id = sub.id
+				"""))
+				count = result.rowcount or 0
+			else:
+				# Упрощённый путь для SQLite.
+				stmt = (
+					update(Car)
+					.where(Car.origin_url.notin_(active_origin_urls))
+					.where(Car.is_sold == False)  # noqa: E712
+					.where(Car.origin_id.like("iaai:%"))
+					.values(is_sold=True)
+				)
+				result = session.execute(stmt)
+				count = result.rowcount or 0
+
+			if count:
+				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
+			return count
+
+	def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все известные origin_id для заданного префикса.
+
+		Использует yield_per для потоковой загрузки при большом количестве записей.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
--- a/iaai_scraper/storage/enums.py
+++ b/iaai_scraper/storage/enums.py
@@ -18,15 +18,6 @@ BODY_TYPE_ENUM_VALUES = (
 )
 COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
 ORIGIN_ENUM_VALUES = (
-	"TAU",
-	"CARSENSOR",
-	"HANAMARU",
-	"ENCAR",
-	"KURUMA_TRADER",
-	"ASNET",
-	"KABABA",
-	"ACV",
-	"COPART",
 	"IAAI",
 	"NA",
 )
--- a/iaai_scraper/storage/models.py
+++ b/iaai_scraper/storage/models.py
@@ -23,6 +23,7 @@ class Car(Base):
 	__tablename__ = "cars"
 	__table_args__ = (
 		Index("ix_cars_brand_model", "brand", "model"),
+		Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
 	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -30,21 +31,21 @@ class Car(Base):
 	model: Mapped[str] = mapped_column(String(50), nullable=False)
 	year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
-	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD")
+	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
-	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True)
-	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True)
-	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True)
-	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER")
+	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
+	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True)
+	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True)
+	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER")
 	engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
-	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
 	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
-	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA")
+	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
 	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
 	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
 	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
--- a/iaai_scraper/worker/tasks.py
+++ b/iaai_scraper/worker/tasks.py
@@ -1,8 +1,10 @@
-# Celery-задачи для синхронизации автомобилей и листинга IAAI.
+# Задачи Celery для синхронизации автомобилей и листинга IAAI.

 from concurrent.futures import ThreadPoolExecutor
-import json
 import logging
+from threading import Event, Thread
+import time
+import uuid

 from celery import shared_task
 from redis import Redis
@@ -17,9 +19,7 @@ SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"


 def _run_browser_job(func, *args, **kwargs):
-    # Playwright Sync API нельзя запускать в потоке с активным asyncio loop.
-    # Celery/зависимости могут поднимать loop в worker-процессе, поэтому
-    # браузерный код выполняем в отдельном thread без loop.
+    # Браузерный код запускаем в отдельном потоке без активного loop.
    with ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser") as executor:
        future = executor.submit(func, *args, **kwargs)
        return future.result()
@@ -27,7 +27,7 @@ def _run_browser_job(func, *args, **kwargs):

 def _sync_listing_lock_ttl_seconds() -> int:
    settings = Settings()
-    # Небольшой запас к hard time limit задачи, чтобы lock самоснимался после сбоев.
+    # Небольшой запас к лимиту времени, чтобы lock снимался после сбоев.
    return max(settings.celery.task_time_limit + 120, 300)


@@ -40,35 +40,70 @@ def _get_redis() -> Redis:
    return Redis.from_url(settings.redis.url, decode_responses=True)


-def _release_lock_if_owner(redis_client: Redis, key: str, owner: str) -> None:
+def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
    try:
-        current_owner = redis_client.get(key)
-        if current_owner == owner:
-            redis_client.delete(key)
+        return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
    except Exception as exc:
-        logger.warning("Failed to release lock %s: %s", key, exc)
-
-
-def _has_other_active_sync_listing_task(task) -> bool:
-    try:
-        inspector = task.app.control.inspect(timeout=1.0)
-        active_map = inspector.active() or {}
-    except Exception as exc:
-        logger.warning("Failed to inspect active tasks: %s", exc)
+        logger.warning("Failed to acquire lock %s", key, exc_info=True)
        return False

-    current_task_id = task.request.id
-    for worker_tasks in active_map.values():
-        for item in worker_tasks or []:
-            name = str(item.get("name") or "")
-            task_id = str(item.get("id") or "")
-            if (
-                name == "iaai_scraper.worker.tasks.sync_listing_task"
-                and task_id
-                and task_id != current_task_id
-            ):
-                return True
-    return False
+
+def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None:
+    try:
+        refreshed = redis_client.eval(
+            """
+            if redis.call('GET', KEYS[1]) == ARGV[1] then
+                return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
+            end
+            return 0
+            """,
+            1,
+            key,
+            owner_token,
+            int(ttl_seconds),
+        )
+        return bool(refreshed)
+    except Exception as exc:
+        logger.warning("Failed to refresh lock %s", key, exc_info=True)
+        return None
+
+
+def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None:
+    try:
+        redis_client.eval(
+            """
+            if redis.call('GET', KEYS[1]) == ARGV[1] then
+                return redis.call('DEL', KEYS[1])
+            end
+            return 0
+            """,
+            1,
+            key,
+            owner_token,
+        )
+    except Exception as exc:
+        logger.warning("Failed to release lock %s", key, exc_info=True)
+
+
+def _start_lock_heartbeat(
+    redis_client: Redis,
+    key: str,
+    owner_token: str,
+    ttl_seconds: int,
+) -> tuple[Event, Thread]:
+    stop_event = Event()
+    interval_seconds = max(5.0, min(30.0, ttl_seconds / 3))
+
+    def _heartbeat() -> None:
+        while not stop_event.wait(interval_seconds):
+            refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds)
+            if refreshed is False:
+                logger.warning("Lost sync_listing lock ownership for %s", owner_token)
+                return
+
+    thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True)
+    thread.start()
+    return stop_event, thread


@shared_task(
@@ -98,7 +133,7 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
        }

    except Exception as exc:
-        logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc)
+        logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True)
        raise self.retry(exc=exc)


@@ -121,44 +156,15 @@ def sync_listing_task(
    persistence = _get_persistence()
    persistence.create_tables()
    task_id = self.request.id or "unknown"
+    owner_token = f"{task_id}:{uuid.uuid4().hex}"
    redis_client = _get_redis()

    lock_acquired = False
    lock_ttl = _sync_listing_lock_ttl_seconds()
-    try:
-        lock_acquired = bool(
-            redis_client.set(
-                SYNC_LISTING_LOCK_KEY,
-                task_id,
-                nx=True,
-                ex=lock_ttl,
-            )
-        )
-    except Exception as exc:
-        logger.warning("Failed to acquire sync lock in Redis: %s", exc)
+    heartbeat_stop: Event | None = None
+    heartbeat_thread: Thread | None = None

-    if not lock_acquired:
-        # Возможен stale lock после рестарта worker. Если активного sync_listing нет —
-        # снимаем lock и пытаемся взять его заново.
-        if not _has_other_active_sync_listing_task(self):
-            try:
-                stale_owner = redis_client.get(SYNC_LISTING_LOCK_KEY)
-                if stale_owner:
-                    logger.warning(
-                        "Removing stale sync lock held by task %s",
-                        stale_owner,
-                    )
-                redis_client.delete(SYNC_LISTING_LOCK_KEY)
-                lock_acquired = bool(
-                    redis_client.set(
-                        SYNC_LISTING_LOCK_KEY,
-                        task_id,
-                        nx=True,
-                        ex=lock_ttl,
-                    )
-                )
-            except Exception as exc:
-                logger.warning("Failed to recover stale sync lock: %s", exc)
+    lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)

    if not lock_acquired:
        logger.info("sync_listing_task skipped: another sync is already running")
@@ -169,6 +175,12 @@ def sync_listing_task(
        }

    try:
+        heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
+            redis_client,
+            SYNC_LISTING_LOCK_KEY,
+            owner_token,
+            lock_ttl,
+        )
        self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
        def _job():
            with IAAIScraper() as scraper:
@@ -198,8 +210,12 @@ def sync_listing_task(
        return {"status": "success", **summary}

    except Exception as exc:
-        logger.error("sync_listing_task failed: %s", exc)
+        logger.error("sync_listing_task failed: %s", exc, exc_info=True)
        raise self.retry(exc=exc)
    finally:
+        if heartbeat_stop is not None:
+            heartbeat_stop.set()
+        if heartbeat_thread is not None:
+            heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
        if lock_acquired:
-            _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, task_id)
+            _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token)
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,6 +19,7 @@ dependencies = [
  "redis>=5.2.0",
  "sqlalchemy>=2.0.32",
  "uvicorn>=0.34.0",
+  "urllib3>=2.0.0",
 ]

 [project.optional-dependencies]
--- a/tests/test_db.py
+++ b/tests/test_db.py
@@ -92,16 +92,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
        self.assertEqual(len(images), 1)
        self.assertIn("imageKeys=2", images[0].fullres_image)

-    def test_persistence_ignores_non_db_fields(self) -> None:
-        record = self._record("1000")
-
-        result = self.persistence.upsert_car(record)
-
-        self.assertEqual(result["action"], "inserted")
-        with self.persistence.session_scope() as session:
-            car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one()
-        self.assertEqual(car.origin_id, "1000")
-
    def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
        first_run_id = self.persistence.start_sync_run("lane-a")
        second_run_id = self.persistence.start_sync_run("lane-b")
--- a/tests/test_listing.py
+++ b/tests/test_listing.py
@@ -31,12 +31,6 @@ class TestListingUnit(unittest.TestCase):
        page = _FakePage({})
        self.assertFalse(ListingCollector._has_next_page(page))

-    def test_constructor_with_settings_and_pacer(self) -> None:
-        settings = Settings()
-        pacer = HumanPacer(settings)
-        collector = ListingCollector(settings, pacer)
-        self.assertIsNotNone(collector)
-

 if __name__ == "__main__":
    unittest.main()
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -13,15 +13,15 @@ class TestVehicleParserUnit(unittest.TestCase):
        dom_text = """
        Stock #:
        45089484
-        VIN (Status):
-        1HGCM82633A123456 (OK)
        Primary Damage:
        Front End
+        Odometer:
+        50,123 mi (Actual)
        """
        result = self.parser._parse_dom_key_value_pairs(dom_text)
        self.assertEqual(result.get("lot_number"), "45089484")
-        self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)")
        self.assertEqual(result.get("primary_damage"), "Front End")
+        self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")

    def test_parse_title_for_year_make_model(self) -> None:
        parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
--- a/tests/test_scraper.py
+++ b/tests/test_scraper.py
@@ -51,15 +51,12 @@ class TestScraperSync(unittest.TestCase):
        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=2)
        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set()))

        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
-        page = MagicMock()
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=page)
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
+        })

        result = scraper.sync_listing()

@@ -67,8 +64,7 @@ class TestScraperSync(unittest.TestCase):
        self.assertEqual(result["cars_failed"], 0)
        self.assertIn("trace_id", result)
        self.assertIn("elapsed_seconds", result)
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
-        page.close.assert_called_once()
+        scraper.sync_batch.assert_called_once()

    def test_sync_listing_respects_limit(self) -> None:
        scraper = self._make_scraper()
@@ -76,38 +72,25 @@ class TestScraperSync(unittest.TestCase):
        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=3)
        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
-        scraper.collect_listing = MagicMock(return_value={
-            "vehicle_urls": [
-                "https://www.iaai.com/VehicleDetail/222~US",
-                "https://www.iaai.com/VehicleDetail/333~US",
-            ]
+
+        # Проверка пути only_new с limit.
+        scraper._collect_listing_iterative = MagicMock(return_value=(
+            ["https://www.iaai.com/VehicleDetail/222~US"],
+            ["https://www.iaai.com/VehicleDetail/222~US",
+             "https://www.iaai.com/VehicleDetail/333~US"],
+            {"vehicle_urls": [], "pages_collected": 1, "early_stopped": False},
+            0,
+        ))
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
        })
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=MagicMock())

        scraper.sync_listing(limit=1)

-        self.assertEqual(scraper._scrape_on_page.call_count, 1)
-
-    def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
-        scraper = self._make_scraper()
-
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=4)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
-        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
-        scraper._get_page = MagicMock(return_value=MagicMock())
-
-        result = scraper.sync_listing()
-
-        self.assertEqual(result["cars_upserted"], 0)
+        # Должен уйти только один URL.
+        scraper.sync_batch.assert_called_once()
+        batch_urls = scraper.sync_batch.call_args[0][0]
+        self.assertEqual(len(batch_urls), 1)

    def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
        scraper = self._make_scraper()
@@ -115,9 +98,10 @@ class TestScraperSync(unittest.TestCase):
        scraper.persistence.create_tables = MagicMock()
        scraper.persistence.start_sync_run = MagicMock(return_value=5)
        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"})
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"})
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
+            {"https://www.iaai.com/VehicleDetail/111~US"},
+            {"iaai:222"},
+        ))

        scraper.collect_listing = MagicMock(return_value={
            "vehicle_urls": [
@@ -126,26 +110,34 @@ class TestScraperSync(unittest.TestCase):
                "https://www.iaai.com/VehicleDetail/333~US",  # new
            ]
        })
-        page = MagicMock()
-        scraper._get_page = MagicMock(return_value=page)
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")})
+        # Возвращаем результат для одного нового авто.
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
+        })

        result = scraper.sync_listing(only_new=True)

        self.assertEqual(result["skipped_existing"], 2)
        self.assertEqual(result["cars_upserted"], 1)
-        self.assertEqual(scraper._scrape_on_page.call_count, 1)
-        scraper.persistence.get_existing_origin_urls.assert_called_once()
-        scraper.persistence.get_existing_origin_ids.assert_called_once()
+        # В batch должен попасть только новый URL.
+        scraper.sync_batch.assert_called_once()
+        batch_urls = scraper.sync_batch.call_args[0][0]
+        self.assertEqual(len(batch_urls), 1)
+        self.assertIn("333", batch_urls[0])
+        scraper.persistence.get_existing_urls_and_ids.assert_called_once()

    def test_close_resets_browser_state(self) -> None:
        scraper = self._make_scraper()
+        http_pool = MagicMock()
+        scraper._http_pool = http_pool
        scraper.context = MagicMock()
        scraper.browser = MagicMock()
        scraper.playwright = MagicMock()

        scraper.close()

+        http_pool.clear.assert_called_once()
+        self.assertIsNone(scraper._http_pool)
        self.assertIsNone(scraper.context)
        self.assertIsNone(scraper.browser)
        self.assertIsNone(scraper.playwright)
@@ -172,6 +164,11 @@ class TestScraperSync(unittest.TestCase):
                "https://www.iaai.com/VehicleDetail/999~US",
            )

+    def test_is_protection_or_network_error_detects_known_signals(self) -> None:
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
+

 if __name__ == "__main__":
    unittest.main()
--- a/tests/test_worker_tasks.py
+++ b/tests/test_worker_tasks.py
@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.worker import tasks
+
+
+class TestWorkerTaskLockHelpers(unittest.TestCase):
+    def test_acquire_lock_returns_true_on_success(self) -> None:
+        redis_client = MagicMock()
+        redis_client.set.return_value = True
+
+        acquired = tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)
+
+        self.assertTrue(acquired)
+        redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
+
+    def test_refresh_lock_if_owner_extends_ttl(self) -> None:
+        redis_client = MagicMock()
+        redis_client.eval.return_value = 1
+
+        refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)
+
+        self.assertTrue(refreshed)
+        redis_client.eval.assert_called_once()
+
+    def test_release_lock_if_owner_uses_owner_token(self) -> None:
+        redis_client = MagicMock()
+
+        tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
+
+        redis_client.eval.assert_called_once()
+        args = redis_client.eval.call_args[0]
+        self.assertEqual(args[1], 1)
+        self.assertEqual(args[2], "lock:key")
+        self.assertEqual(args[3], "owner-token")
+
+    def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=False):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            get_redis.return_value = MagicMock()
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        persistence.create_tables.assert_called_once()
+        self.assertEqual(result["status"], "skipped")
+        self.assertEqual(result["reason"], "sync_already_running")
+
+    def test_sync_listing_task_releases_owned_lock(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 7,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 4,
+                 "skipped_existing": 1,
+                 "elapsed_seconds": 1.25,
+             }):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            tasks.sync_listing_task.push_request(id="task-123")
+            try:
+                result = tasks.sync_listing_task.run(make="Toyota")
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        stop_event.set.assert_called_once()
+        heartbeat_thread.join.assert_called_once()
+        release_lock.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()