import json
import logging
import os
import random
import re
import signal
import time
import uuid
import html as html_module
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import TimeoutError as FuturesTimeoutError
from datetime import datetime, timezone
from pathlib import Path
from threading import Event, Thread
from typing import Any, Callable
from urllib.parse import urlsplit, urlunsplit
from urllib.request import Request, urlopen

import urllib3

from playwright.sync_api import Error as PlaywrightError
from playwright.sync_api import BrowserContext, Page, sync_playwright
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError

from .browser import BrowserFactory, HumanPacer, NetworkCapture
from .core.config import Settings, settings, parse_listing_segments
from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError
from .core.logs import set_trace_id, setup_logging
from .core.retry import retryable
from .core.runtime_config import RuntimeConfig
from .core.utils import save_to_json
from .discovery import AlgoliaDiscoveryError, discover_vehicle_hits_from_algolia
from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError
from .parsing.mapper import CarMapper
from .parsing.parser import VehicleParser
from .storage.db import PersistenceService
from .browser.listing import ListingCollector, ListingPageResult
from .storage.schemas import CarRecord

logger = logging.getLogger("dubizzle_scraper.scraper")
VEHICLE_ID_RE = re.compile(
    r"(?:/VehicleDetail/(?P<veh_id>\d+)(?:~[A-Z]{2})?)|(?:---(?P<slug_id>[a-f0-9]{32})/?$)|(?:/ad-(?P<ad_id>\d+)/?)",
    re.IGNORECASE,
)
HTML_TAG_RE = re.compile(r"<[^>]+>")
SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
NEXT_DATA_RE = re.compile(
    r'<script[^>]+id=["\']__NEXT_DATA__["\'][^>]*type=["\']application/json["\'][^>]*>(.*?)</script>',
    re.IGNORECASE | re.DOTALL,
)

# Таймауты операций страницы.
_PAGE_COLLECT_TIMEOUT_S = 90
_PAGE_NEXT_TIMEOUT_S = 60
# Ротация контекста выключена по умолчанию.
_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("DUBIZZLE_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0)
_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("DUBIZZLE_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3)
_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120
_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2


class PageOperationTimeoutError(Exception):
    """Browser page operation exceeded per-op watchdog timeout."""


class _PageOpWatchdog:
    """SIGALRM-based watchdog.

    Работает только в главном потоке процесса (Celery prefork child — это ок).
    В других контекстах — no-op.
    """

    def __init__(self, seconds: int, label: str) -> None:
        self.seconds = max(1, int(seconds))
        self.label = label
        self._old_handler = None
        self._active = False

    def _handler(self, signum, frame):  # noqa: ARG002
        raise PageOperationTimeoutError(
            f"Page operation '{self.label}' exceeded {self.seconds}s watchdog"
        )

    def __enter__(self):
        import threading as _threading
        if _threading.current_thread() is not _threading.main_thread():
            return self
        if not hasattr(signal, "SIGALRM"):
            return self
        try:
            self._old_handler = signal.signal(signal.SIGALRM, self._handler)
            signal.alarm(self.seconds)
            self._active = True
        except (ValueError, OSError):
            # signal можно выставлять только из main thread; в остальном пропускаем.
            self._active = False
        return self

    def __exit__(self, exc_type, exc, tb):
        if not self._active:
            return False
        try:
            signal.alarm(0)
            if self._old_handler is not None:
                signal.signal(signal.SIGALRM, self._old_handler)
        except (ValueError, OSError):
            pass
        return False


class _ProgressHeartbeat:
    """Периодически пульсует progress во время долгой навигации."""

    def __init__(self, report_progress: Callable[[str, Any], None], stage: str, interval_s: float = 15.0, **meta: Any) -> None:
        self._report_progress = report_progress
        self._stage = stage
        self._interval_s = max(5.0, float(interval_s))
        self._meta = meta
        self._stop = Event()
        self._thread: Thread | None = None

    def _run(self) -> None:
        while not self._stop.wait(self._interval_s):
            self._report_progress(self._stage, heartbeat_only=True, **self._meta)

    def __enter__(self):
        self._thread = Thread(target=self._run, name=f"progress-heartbeat-{self._stage}", daemon=True)
        self._thread.start()
        return self

    def __exit__(self, exc_type, exc, tb):
        self._stop.set()
        if self._thread is not None:
            self._thread.join(timeout=1.0)
        return False


class DUBIZZLEScraper:

    @staticmethod
    def _is_protection_or_network_error(exc: Exception) -> bool:
        message = str(exc).lower()
        signals = (
            "captcha",
            "antibot",
            "blocked",
            "challenge",
            "ns_error_net_interrupt",
            "navigation",
            "timeout",
            "403",
            "429",
        )
        return any(signal in message for signal in signals)

    @staticmethod
    def _html_to_text(html: str) -> str:
        if not html:
            return ""
        cleaned = SCRIPT_STYLE_RE.sub(" ", html)
        text = HTML_TAG_RE.sub(" ", cleaned)
        text = html_module.unescape(text)
        return re.sub(r"\s+", " ", text).strip()

    @staticmethod
    def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None:
        dom_hints = parsed.get("dom_hints", {}) or {}
        access_notes = parsed.get("access_notes", {}) or {}
        summary = parsed.get("vehicle_summary", {}) or {}

        has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model"))

        if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity:
            raise AntiBotDetectedError(f"DUBIZZLE anti-bot detected for {vehicle_url}")

        if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity:
            raise AntiBotDetectedError(f"DUBIZZLE blocked or challenged request for {vehicle_url}")

        if not has_identity:
            raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}")

    @staticmethod
    def _extract_origin_id_from_url(vehicle_url: str) -> str | None:
        match = VEHICLE_ID_RE.search(vehicle_url)
        if not match:
            return None
        return match.group("veh_id") or match.group("slug_id") or match.group("ad_id")

    @staticmethod
    def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None:
        raw_id = DUBIZZLEScraper._extract_origin_id_from_url(vehicle_url)
        if not raw_id:
            return None
        return f"dubizzle:{raw_id}"

    @staticmethod
    def _normalize_vehicle_url(vehicle_url: str) -> str:
        try:
            parts = urlsplit(vehicle_url)
            return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
        except Exception:
            return vehicle_url

    def __init__(self, runtime_settings: Settings | None = None) -> None:
        self.settings = runtime_settings or settings
        setup_logging(self.settings.log_level, self.settings.log_file)
        self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file)
        self.trace_id = uuid.uuid4().hex[:12]
        set_trace_id(self.trace_id)
        self.playwright = None
        self.browser = None
        self.context: BrowserContext | None = None
        self.browser_factory = BrowserFactory(self.settings)
        self.pacer = HumanPacer(self.settings)
        self.listing_collector = ListingCollector(self.settings, self.pacer)
        self.vehicle_parser = VehicleParser()
        self.car_mapper = CarMapper()
        self.persistence = PersistenceService(self.settings)
        self._shutdown_requested = False
        self._progress_callback: Callable[[str, dict], None] | None = None
        # HTTP pool: при parallel_tabs=24 и concurrency=4 пик ≈ 96 конкурентных сокетов;
        # даём запас до 256, чтобы не упираться в PoolError под всплесками PX/retry.
        proxy_url = self.settings.proxy.server
        if proxy_url:
            _proxy_kwargs: dict = {
                "num_pools": 4,
                "maxsize": 64,
                "block": False,
                "retries": False,
                "timeout": urllib3.Timeout(connect=5, read=20),
            }
            if self.settings.proxy.username:
                _proxy_kwargs["proxy_headers"] = urllib3.make_headers(
                    proxy_basic_auth=f"{self.settings.proxy.username}:{self.settings.proxy.password or ''}"
                )
            self._http_pool: urllib3.PoolManager = urllib3.ProxyManager(proxy_url, **_proxy_kwargs)
            logger.info("HTTP fast-path using proxy: %s", proxy_url)
        else:
            self._http_pool = urllib3.PoolManager(
                num_pools=4,
                maxsize=64,
                block=False,
                retries=False,
                timeout=urllib3.Timeout(connect=5, read=20),
            )

    def _new_trace_id(self, prefix: str) -> str:
        trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}"
        self.trace_id = trace_id
        set_trace_id(trace_id)
        return trace_id

    def _reload_runtime_config(self) -> None:
        """Reload runtime config from file to apply include/exclude changes without restart."""
        try:
            self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file)
        except Exception:
            logger.warning("Failed to reload runtime config, keeping previous values", exc_info=True)

    def set_progress_callback(self, callback: Callable[[str, dict], None]) -> None:
        self._progress_callback = callback

    def _report_progress(self, stage: str, **meta: Any) -> None:
        if self._progress_callback is not None:
            try:
                self._progress_callback(stage, meta)
            except Exception:
                pass

    def __enter__(self) -> "DUBIZZLEScraper":
        return self

    def __exit__(self, exc_type, exc, tb) -> None:
        self.close()

    def close(self) -> None:
        if self.context is not None:
            try:
                self.context.close()
            except PlaywrightError:
                pass
            finally:
                self.context = None
        if self.browser is not None:
            try:
                self.browser.close()
            except PlaywrightError:
                pass
            finally:
                self.browser = None
        if self.playwright is not None:
            try:
                self.playwright.stop()
            except PlaywrightError:
                pass
            finally:
                self.playwright = None
        if getattr(self, "_http_pool", None) is not None:
            try:
                self._http_pool.clear()
            except Exception:
                pass
            finally:
                self._http_pool = None

    def _new_context(self) -> BrowserContext:
        if self.browser is None:
            if self.playwright is None:
                self.playwright = sync_playwright().start()
            self.browser = self.browser_factory.create_browser(self.playwright)
        if self.context:
            try:
                self.context.close()
            except PlaywrightError:
                pass
        self.context = self.browser_factory.create_context(self.browser)
        return self.context

    def init_db(self):
        self.persistence.create_tables()
        return {"status": "ok", "database_url": self.settings.database.url}

    def _warmup_visit(self, page: Page) -> None:
        try:
            logger.info("Warmup: visiting homepage to pass anti-bot challenge...")
            page.goto(self.settings.home_url, wait_until="commit", timeout=30_000)
            try:
                page.wait_for_load_state("domcontentloaded", timeout=6_000)
            except PlaywrightTimeoutError:
                pass
            try:
                page.wait_for_function("() => document.title && document.title.length > 3", timeout=4_000)
            except PlaywrightTimeoutError:
                pass
            time.sleep(0.3)
            logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50])
        except Exception as e:
            logger.warning("Warmup visit failed: %s — continuing anyway", e)
            time.sleep(1.5)

    def _get_page_with_warmup(self) -> Page:
        context = self._new_context()
        page = context.new_page()
        self._warmup_visit(page)
        return page

    def _dedupe_urls(self, raw_urls: list[str]) -> list[str]:
        vehicle_urls: list[str] = []
        seen: set[str] = set()
        for raw in raw_urls:
            normalized = self._normalize_vehicle_url(raw)
            if normalized not in seen:
                seen.add(normalized)
                vehicle_urls.append(normalized)
        return vehicle_urls

    def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]:
        url_to_origin_id = {
            url: self._extract_db_origin_id_from_url(url)
            for url in vehicle_urls
        }
        candidate_origin_ids = [oid for oid in url_to_origin_id.values() if oid]
        existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(
            vehicle_urls, candidate_origin_ids,
        )
        known_urls = {
            url for url in vehicle_urls
            if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids)
        }
        skipped = len(known_urls)
        if skipped:
            logger.info("Filtering already known vehicles: skipped %d", skipped)
        new_urls = [url for url in vehicle_urls if url not in known_urls]
        return new_urls, skipped

    def _extract_page_urls(
        self,
        page_result: ListingPageResult,
        all_raw_urls: list[str],
        seen_urls: set[str],
        all_listing_origin_urls: set[str] | None = None,
    ) -> list[str]:
        page_urls: list[str] = []
        for item in page_result.vehicle_links:
            normalized = self._normalize_vehicle_url(item.href)
            all_raw_urls.append(item.href)
            if normalized and all_listing_origin_urls is not None:
                all_listing_origin_urls.add(normalized)
            if normalized and normalized not in seen_urls:
                seen_urls.add(normalized)
                page_urls.append(normalized)
        return page_urls

    @staticmethod
    def _build_segment_listing_url(base_url: str, make: str | None) -> str:
        """Построить URL листинга для сегмента Dubizzle.

        Для актуальных URL используем path-формат `/motors/used-cars/{make}/`.
        Для legacy URL (`Vehiclelisting`) оставляем query-параметр `?Make=`.
        """
        if not make:
            return base_url
        if "motors/used-cars" in base_url.lower() and "vehiclelisting" not in base_url.lower():
            normalized_base = base_url.rstrip("/")
            make_slug = make.strip().lower().replace(" ", "-")
            return f"{normalized_base}/{make_slug}/"
        sep = "&" if "?" in base_url else "?"
        return f"{base_url}{sep}Make={make.replace(' ', '%20')}"

    def _recover_empty_listing_page(
        self,
        page: Page,
        *,
        page_number: int,
        all_raw_urls: list[str],
        seen_urls: set[str],
        all_listing_origin_urls: set[str] | None = None,
        listing_url: str | None = None,
    ) -> tuple[ListingPageResult, list[str]]:
        if page_number == 1:
            logger.warning("Page 1 returned 0 links — retrying listing open...")
            time.sleep(3)
            self.listing_collector.open_cars_listing(page, url_override=listing_url)
            page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
            return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls)

        logger.warning(
            "Page %d returned 0 links — retrying current page before stopping pagination",
            page_number,
        )
        try:
            page.reload(wait_until="domcontentloaded", timeout=30_000)
        except Exception as exc:
            logger.debug("Page %d reload failed during empty-page recovery: %s", page_number, exc)
        page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
        return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls)

    def _open_listing_page(
        self,
        *,
        make: str | None,
        model: str | None,
        listing_url: str | None = None,
        year_min: int | None = None,
        year_max: int | None = None,
    ) -> tuple[Page, dict[str, str | int | None]]:
        page = self._get_page_with_warmup()
        try:
            self.listing_collector.open_cars_listing(page, url_override=listing_url)
            applied_filters = self.listing_collector.apply_filters(
                page,
                make=make,
                model=model,
                year_min=year_min,
                year_max=year_max,
            )
        except Exception:
            page.close()
            raise
        return page, applied_filters

    def _reopen_listing_and_resume(
        self,
        *,
        target_page_number: int,
        make: str | None,
        model: str | None,
        listing_url: str | None = None,
        year_min: int | None = None,
        year_max: int | None = None,
        max_nav_pages: int = 10,
    ) -> tuple[Page, dict[str, str | int | None]]:
        # Ограничиваем глубину навигации: если до цели > max_nav_pages кликов — не пытаемся.
        if target_page_number > max_nav_pages + 1:
            raise ListingResumeError(
                f"Cannot resume at page {target_page_number}: "
                f"exceeds max navigation depth ({max_nav_pages} pages)"
            )

        self._report_progress(
            "listing_resume_started",
            target_page=target_page_number,
            max_nav_pages=max_nav_pages,
        )

        page, applied_filters = self._open_listing_page(
            make=make,
            model=model,
            listing_url=listing_url,
            year_min=year_min,
            year_max=year_max,
        )

        self._report_progress(
            "listing_resume_opened",
            target_page=target_page_number,
        )

        for expected_page in range(2, target_page_number + 1):
            self._report_progress(
                "listing_resume_progress",
                current_page=expected_page - 1,
                target_page=target_page_number,
                next_page_number=expected_page,
            )

            with _ProgressHeartbeat(
                self._report_progress,
                "listing_resume_progress",
                current_page=expected_page - 1,
                target_page=target_page_number,
                next_page_number=expected_page,
            ):
                next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=expected_page)

            if not next_ok:
                self._report_progress(
                    "listing_resume_failed",
                    current_page=expected_page - 1,
                    target_page=target_page_number,
                )
                page.close()
                raise ListingResumeError(f"Failed to resume listing at page {target_page_number}")

            self._report_progress(
                "listing_resume_progress",
                current_page=expected_page,
                target_page=target_page_number,
                next_page_number=min(target_page_number, expected_page + 1),
            )

        self._report_progress(
            "listing_resume_completed",
            current_page=target_page_number,
            target_page=target_page_number,
        )
        logger.warning("Listing resumed at page %d after recovery", target_page_number)
        return page, applied_filters

    def _open_listing_for_stream(
        self,
        *,
        make: str | None,
        model: str | None,
        listing_url: str | None = None,
        year_min: int | None = None,
        year_max: int | None = None,
    ) -> tuple[Page, dict[str, str | int | None]]:
        # Всегда открываем с page 1. Page-level resume убран: эфемерные URL и короткие
        # сегменты делали pagination-resume хрупким. Bootstrap прогресс сохраняется
        # на уровне сегментов в worker/tasks.py (_save_last_completed_segment).
        return self._open_listing_page(
            make=make,
            model=model,
            listing_url=listing_url,
            year_min=year_min,
            year_max=year_max,
        )

    def collect_listing(
        self,
        make: str | None = None,
        model: str | None = None,
        known_origin_ids: set[str] | None = None,
        max_duration_seconds: float | None = None,
    ):
        result = discover_vehicle_hits_from_algolia(
            settings=self.settings,
            make=make,
            model=model,
            limit=None,
            year_min=None,
            year_max=None,
            listing_url=None,
            known_origin_ids=known_origin_ids,
            max_duration_seconds=max_duration_seconds,
        )
        return {
            "status": "ok",
            "vehicles_collected": len(result.vehicle_urls),
            "vehicle_urls": result.vehicle_urls,
            "early_stopped": result.early_stopped,
            "truncated_by_time_budget": result.truncated_by_time_budget,
            "pages": result.pages,
            "total_hits": result.total_hits,
        }

    @staticmethod
    def _extract_listing_items_from_page(page: Page) -> list[dict[str, Any]]:
        try:
            items = page.evaluate(
                r'''
                () => {
                    const normalizeItem = (item) => {
                        if (!item || typeof item !== 'object') return null;
                        return {
                            url: item.url || null,
                            name: item.name || null,
                            description: item.description || null,
                            brand: item.brand && typeof item.brand === 'object' ? (item.brand.name || null) : null,
                            model: item.model || null,
                            year: item.vehicleModelDate || null,
                            mileage_km: item.mileageFromOdometer && typeof item.mileageFromOdometer === 'object'
                                ? (item.mileageFromOdometer.value ?? null)
                                : null,
                            price_aed: item.offers && typeof item.offers === 'object' ? (item.offers.price ?? null) : null,
                            currency: item.offers && typeof item.offers === 'object' ? (item.offers.priceCurrency || null) : null,
                            location: item.offers && item.offers.areaServed && item.offers.areaServed.address
                                ? (item.offers.areaServed.address.addressLocality || null)
                                : null,
                            image: item.image || null,
                        };
                    };

                    const out = [];
                    const seen = new Set();

                    const pushItem = (item) => {
                        const normalized = normalizeItem(item);
                        if (!normalized || !normalized.url || seen.has(normalized.url)) return;
                        seen.add(normalized.url);
                        out.push(normalized);
                    };

                    for (const el of document.querySelectorAll('script[type="application/ld+json"]')) {
                        try {
                            const data = JSON.parse(el.textContent || '{}');
                            const list = data && data.mainEntity && Array.isArray(data.mainEntity.itemListElement)
                                ? data.mainEntity.itemListElement
                                : [];
                            for (const entry of list) {
                                pushItem(entry && typeof entry === 'object' ? (entry.item || entry) : null);
                            }
                        } catch (_) {}
                    }

                    if (out.length) return out;

                    const nextDataEl = document.querySelector('#__NEXT_DATA__');
                    if (!nextDataEl) return out;
                    try {
                        const nextData = JSON.parse(nextDataEl.textContent || '{}');
                        const serialized = JSON.stringify(nextData);
                        const matches = serialized.match(/https:\/\/[^\"]+\/motors\/used-cars\/[^\"]+?(?:---[a-f0-9]{32}|\/ad-\d+\/?)/ig) || [];
                        for (const url of matches) {
                            if (seen.has(url)) continue;
                            seen.add(url);
                            out.push({
                                url,
                                name: null,
                                description: null,
                                brand: null,
                                model: null,
                                year: null,
                                mileage_km: null,
                                price_aed: null,
                                currency: 'AED',
                                location: null,
                                image: null,
                            });
                        }
                    } catch (_) {}
                    return out;
                }
                '''
            )
        except Exception:
            return []
        if not isinstance(items, list):
            return []
        return [item for item in items if isinstance(item, dict) and item.get("url")]

    def _build_record_from_listing_item(self, item: dict[str, Any]) -> CarRecord:
        url = str(item.get("url") or "").strip()
        image = str(item.get("image") or "").strip()
        summary = {
            "source_url": url,
            "make": item.get("brand"),
            "model": item.get("model"),
            "year": item.get("year"),
            "odometer": item.get("mileage_km"),
            "buy_now": item.get("price_aed"),
            "currency": item.get("currency") or "AED",
            "location": item.get("location"),
            "title": item.get("name"),
            "image_urls": [image] if image else [],
        }
        payload_insights = {
            "vehicle_core": {
                "make": item.get("brand"),
                "model": item.get("model"),
                "year": item.get("year"),
                "odometer": item.get("mileage_km"),
                "location": item.get("location"),
                "title": item.get("name"),
                "country": "AE",
                "selling_type": "STOCK",
            },
            "pricing": {
                "buy_now": item.get("price_aed"),
                "current_bid": None,
                "actual_cash_value": None,
                "currency": item.get("currency") or "AED",
            },
            "damage": {},
            "auction": {"sale_status": "active"},
            "images": {"urls": [image] if image else []},
        }
        return self.car_mapper.map_to_car_record(
            vehicle_url=url,
            vehicle_summary=summary,
            payload_insights=payload_insights,
        )

    @staticmethod
    def _extract_listing_items_with_wait(page: Page, attempts: int = 3) -> list[dict[str, Any]]:
        for attempt in range(max(1, attempts)):
            items = DUBIZZLEScraper._extract_listing_items_from_page(page)
            if items:
                return items
            try:
                page.wait_for_function(
                    r'''() => {
                        const hasLdJson = Array.from(document.querySelectorAll('script[type="application/ld+json"]'))
                            .some((el) => {
                                const text = el.textContent || '';
                                return text.includes('itemListElement') || text.includes('SearchResultsPage');
                            });
                        const nextData = document.querySelector('#__NEXT_DATA__')?.textContent || '';
                        return hasLdJson || nextData.length > 1000;
                    }''',
                    timeout=5_000,
                )
            except Exception:
                pass
            if attempt < attempts - 1:
                try:
                    page.wait_for_timeout(1_500)
                except Exception:
                    time.sleep(1.5)
        return []

    def _build_sync_result_from_listing_items(
        self,
        *,
        listing_items: list[dict[str, Any]],
        make: str | None,
        model: str | None,
        lane: str,
        limit: int | None,
        effective_only_new: bool,
        applied_filters: dict[str, str | int | None],
        listing_url: str | None,
    ) -> dict[str, Any] | None:
        filtered_items = listing_items
        if make:
            filtered_items = [item for item in filtered_items if str(item.get("brand") or "").strip().lower() == make.strip().lower()]
        if model:
            filtered_items = [item for item in filtered_items if str(item.get("model") or "").strip().lower() == model.strip().lower()]

        all_raw_urls = [str(item.get("url")) for item in filtered_items if item.get("url")]
        if not all_raw_urls:
            return None

        pages_info = [{"page_number": 1, "links_found": len(all_raw_urls), "source": "listing_jsonld"}]
        cars_upserted = 0
        cars_failed = 0
        images_upserted = 0
        skipped_existing = 0
        failures: list[dict[str, str]] = []

        records: list[CarRecord] = []
        for item in filtered_items[:limit if limit is not None and limit > 0 else None]:
            try:
                records.append(self._build_record_from_listing_item(item))
            except Exception as exc:
                cars_failed += 1
                failures.append({"vehicle_url": str(item.get("url") or "listing_jsonld"), "error": str(exc)})

        if effective_only_new and records:
            existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(
                [record.origin_url for record in records],
                [record.origin_id for record in records],
            )
            fresh_records: list[CarRecord] = []
            for record in records:
                if record.origin_url in existing_urls or record.origin_id in existing_ids:
                    skipped_existing += 1
                    continue
                fresh_records.append(record)
            records = fresh_records

        if records:
            try:
                db_result = self.persistence.upsert_cars_batch(records)
                cars_upserted += db_result.get("inserted", 0) + db_result.get("updated", 0)
                images_upserted += db_result.get("images_upserted", 0)
            except Exception as exc:
                cars_failed += len(records)
                failures.append({"vehicle_url": "listing_jsonld_batch", "error": str(exc)})

        total = len(records) if effective_only_new else len(all_raw_urls)
        all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url}
        all_listing_origin_ids = {
            origin_id
            for origin_id in (self._extract_db_origin_id_from_url(url) for url in all_raw_urls)
            if origin_id
        }
        return {
            "listing": {
                "status": "ok",
                "listing_url": listing_url or self.settings.listing.cars_url,
                "applied_filters": applied_filters,
                "pages_collected": 1,
                "vehicles_collected": len(all_raw_urls),
                "vehicle_urls": all_raw_urls,
                "early_stopped": False,
                "truncated_by_time_budget": False,
                "pages": pages_info,
                "discovery_source": "listing_jsonld",
            },
            "total": total,
            "skipped_existing": skipped_existing,
            "cars_upserted": cars_upserted,
            "cars_failed": cars_failed,
            "images_upserted": images_upserted,
            "protection_events": 0,
            "failures": failures,
            "all_listing_origin_urls": all_listing_origin_urls,
            "all_listing_origin_ids": all_listing_origin_ids,
        }

    def _sync_listing_streaming(
        self,
        *,
        make: str | None,
        model: str | None,
        lane: str,
        limit: int | None,
        effective_only_new: bool,
        started_at: float,
        listing_url: str | None = None,
        year_min: int | None = None,
        year_max: int | None = None,
    ) -> dict[str, Any]:
        batch_size = self.settings.celery.batch_size
        skipped_existing = 0
        total = 0
        cars_upserted = 0
        cars_failed = 0
        protection_events = 0
        images_upserted = 0
        failures: list[dict[str, str]] = []

        known_origin_ids: set[str] | None = None
        threshold = self.settings.listing.early_stop_threshold
        if effective_only_new and threshold > 0.0:
            try:
                known_origin_ids = self.persistence.get_all_origin_ids_for_lane("dubizzle:")
            except Exception as exc:
                logger.warning("Could not load known origin_ids for early-stop: %s", exc)

        # Legacy only_new path для совместимости с текущими тестами и старым поведением.
        if effective_only_new and (limit is None or limit <= 0) and not listing_url and year_min is None and year_max is None:
            listing_payload = self.collect_listing(
                make=make,
                model=model,
                known_origin_ids=known_origin_ids,
                max_duration_seconds=None,
            )
            raw_urls = list(listing_payload.get("vehicle_urls", []))
            deduped_urls = self._dedupe_urls(raw_urls)
            fresh_urls, page_skipped = self._filter_known_urls(deduped_urls)
            skipped_existing += page_skipped
            total = len(fresh_urls)

            if fresh_urls:
                try:
                    batch_result = self.sync_batch(fresh_urls, lane=lane)
                    cars_upserted += int(batch_result.get("cars_upserted", 0))
                    cars_failed += int(batch_result.get("cars_failed", 0))
                    images_upserted += int(batch_result.get("images_upserted", 0))
                    failures.extend(batch_result.get("failures", []))
                except Exception as exc:
                    cars_failed += len(fresh_urls)
                    failures.append({"vehicle_url": "legacy_only_new", "error": str(exc)})

            all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in raw_urls if url}
            return {
                "listing": {
                    "status": "ok",
                    "listing_url": listing_url or self.settings.listing.cars_url,
                    "applied_filters": {
                        "make": make,
                        "model": model,
                        "year_min": year_min,
                        "year_max": year_max,
                    },
                    "pages_collected": len(listing_payload.get("pages", [])) if isinstance(listing_payload.get("pages"), list) else 0,
                    "vehicles_collected": len(raw_urls),
                    "vehicle_urls": raw_urls,
                    "early_stopped": bool(listing_payload.get("early_stopped", False)),
                    "truncated_by_time_budget": bool(listing_payload.get("truncated_by_time_budget", False)),
                    "pages": listing_payload.get("pages", []),
                },
                "total": total,
                "skipped_existing": skipped_existing,
                "cars_upserted": cars_upserted,
                "cars_failed": cars_failed,
                "images_upserted": images_upserted,
                "protection_events": protection_events,
                "failures": failures,
                "all_listing_origin_urls": all_listing_origin_urls,
            }

        elapsed = max(0.0, time.perf_counter() - started_at)
        remaining_budget = max(60.0, float(self.settings.celery.task_soft_time_limit) - elapsed - 60.0)

        try:
            algolia_result = discover_vehicle_hits_from_algolia(
                settings=self.settings,
                make=make,
                model=model,
                limit=limit,
                year_min=year_min,
                year_max=year_max,
                listing_url=listing_url,
                known_origin_ids=known_origin_ids,
                max_duration_seconds=remaining_budget,
            )
        except AlgoliaDiscoveryError as exc:
            logger.warning("Algolia discovery failed, fallback to browser listing: %s", exc)
            browser_result = self._sync_listing_streaming_browser_fallback(
                make=make,
                model=model,
                lane=lane,
                limit=limit,
                effective_only_new=effective_only_new,
                listing_url=listing_url,
                year_min=year_min,
                year_max=year_max,
            )
            # Если после браузерного fallback всё равно 0 и включён флаг — пробуем sitemap discovery.
            if (
                int(browser_result.get("total") or 0) == 0
                and self.settings.discovery.sitemap_fallback_on_empty
                and make is None
                and model is None
                and year_min is None
                and year_max is None
            ):
                try:
                    logger.warning("Browser fallback returned 0 vehicles, trying sitemap fallback")
                    return self._sync_listing_streaming_sitemap_fallback(
                        lane=lane,
                        limit=limit,
                        effective_only_new=effective_only_new,
                    )
                except Exception as sitemap_exc:
                    logger.warning("Sitemap fallback failed: %s", sitemap_exc)
            return browser_result

        all_raw_urls = list(algolia_result.vehicle_urls)
        all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url}
        all_listing_origin_ids = {
            str(origin_id).strip()
            for origin_id in algolia_result.origin_ids_by_url.values()
            if str(origin_id).strip()
        }
        pages_info = list(algolia_result.pages)
        effective_urls = list(all_raw_urls)

        if effective_only_new and effective_urls:
            candidate_ids = [
                algolia_result.origin_ids_by_url[url]
                for url in effective_urls
                if url in algolia_result.origin_ids_by_url
            ]
            existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(effective_urls, candidate_ids)
            filtered_urls: list[str] = []
            for url in effective_urls:
                oid = algolia_result.origin_ids_by_url.get(url)
                if url in existing_urls or (oid and oid in existing_ids):
                    skipped_existing += 1
                    continue
                filtered_urls.append(url)
            effective_urls = filtered_urls

        if limit is not None and limit > 0:
            effective_urls = effective_urls[:limit]

        total = len(effective_urls)
        logger.info("Algolia listing discovered %d vehicles, processing %d", len(all_raw_urls), total)

        for batch_start in range(0, len(effective_urls), batch_size):
            batch_urls = effective_urls[batch_start: batch_start + batch_size]
            records: list[CarRecord] = []

            for url in batch_urls:
                hit = algolia_result.hit_records.get(url)
                if not hit:
                    cars_failed += 1
                    failures.append({"vehicle_url": url, "error": "Missing Algolia hit for URL"})
                    continue
                try:
                    record = self.car_mapper.map_to_car_record(
                        vehicle_url=url,
                        vehicle_summary=hit,
                        payload_insights={
                            "vehicle_core": {
                                "lot_number": hit.get("id") or hit.get("objectID"),
                                "year": hit.get("year"),
                                "make": hit.get("make"),
                                "model": hit.get("model"),
                                "trim": hit.get("trim") or hit.get("motors_trim"),
                                "odometer": hit.get("kilometers") or hit.get("odometer"),
                                "body_type": hit.get("body_type"),
                                "gearbox": hit.get("transmission_type") or hit.get("transmission"),
                                "drive": hit.get("drive_type"),
                                "fuel_type": hit.get("fuel_type"),
                                "color": hit.get("exterior_color") or hit.get("color"),
                                "seller": hit.get("seller_type"),
                                "location": hit.get("location_name") or hit.get("location"),
                                "title": hit.get("title") or hit.get("name"),
                                "country": "AE",
                                "selling_type": "STOCK",
                            },
                            "pricing": {
                                "buy_now": hit.get("price"),
                                "current_bid": None,
                                "actual_cash_value": None,
                                "currency": hit.get("price_currency") or "AED",
                            },
                            "damage": {},
                            "auction": {"sale_status": hit.get("status")},
                            "images": {
                                "urls": hit.get("photo_mains") or hit.get("photo_thumbnails") or [],
                            },
                        },
                    )
                    records.append(record)
                except Exception as rec_exc:
                    cars_failed += 1
                    failures.append({"vehicle_url": url, "error": str(rec_exc)})

            if not records:
                continue

            try:
                db_result = self.persistence.upsert_cars_batch(records)
                cars_upserted += db_result.get("inserted", 0) + db_result.get("updated", 0)
                images_upserted += db_result.get("images_upserted", 0)
            except Exception as db_exc:
                cars_failed += len(records)
                failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(db_exc)})

        return {
            "listing": {
                "status": "ok",
                "listing_url": listing_url or self.settings.listing.cars_url,
                "applied_filters": {
                    "make": make,
                    "model": model,
                    "year_min": year_min,
                    "year_max": year_max,
                },
                "pages_collected": len(pages_info),
                "vehicles_collected": len(all_raw_urls),
                "vehicle_urls": all_raw_urls,
                "early_stopped": algolia_result.early_stopped,
                "truncated_by_time_budget": algolia_result.truncated_by_time_budget,
                "pages": pages_info,
                "total_hits": algolia_result.total_hits,
            },
            "total": total,
            "skipped_existing": skipped_existing,
            "cars_upserted": cars_upserted,
            "cars_failed": cars_failed,
            "images_upserted": images_upserted,
            "protection_events": protection_events,
            "failures": failures,
            "all_listing_origin_urls": all_listing_origin_urls,
            "all_listing_origin_ids": all_listing_origin_ids,
        }

    def _sync_listing_streaming_sitemap_fallback(
        self,
        *,
        lane: str,
        limit: int | None,
        effective_only_new: bool,
    ) -> dict[str, Any]:
        try:
            discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=self.settings)
        except SitemapDiscoveryError as exc:
            raise RuntimeError(f"Sitemap discovery failed: {exc}") from exc

        all_urls = list(discovery_result.vehicle_urls)
        urls = list(all_urls)
        skipped_existing = 0
        if effective_only_new and urls:
            urls, skipped_existing = self._filter_known_urls(urls)

        if limit is not None and limit > 0:
            urls = urls[:limit]

        cars_upserted = 0
        cars_failed = 0
        images_upserted = 0
        failures: list[dict[str, str]] = []
        batch_size = self.settings.celery.batch_size

        for i in range(0, len(urls), batch_size):
            chunk = urls[i:i + batch_size]
            try:
                result = self.sync_batch(chunk, lane=lane)
                cars_upserted += int(result.get("cars_upserted", 0))
                cars_failed += int(result.get("cars_failed", 0))
                images_upserted += int(result.get("images_upserted", 0))
                failures.extend(result.get("failures", []))
            except Exception as exc:
                cars_failed += len(chunk)
                failures.append({"vehicle_url": f"sitemap_batch_{i}", "error": str(exc)})

        all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_urls if url}
        all_listing_origin_ids = {
            origin_id
            for origin_id in (self._extract_db_origin_id_from_url(url) for url in all_urls)
            if origin_id
        }
        return {
            "listing": {
                "status": "ok",
                "listing_url": self.settings.listing.cars_url,
                "applied_filters": {"make": None, "model": None, "year_min": None, "year_max": None},
                "pages_collected": 1,
                "vehicles_collected": len(all_urls),
                "vehicle_urls": all_urls,
                "early_stopped": False,
                "truncated_by_time_budget": False,
                "pages": [{"page_number": 1, "links_found": len(all_urls)}],
                "discovery_source": "sitemap",
            },
            "total": len(urls),
            "skipped_existing": skipped_existing,
            "cars_upserted": cars_upserted,
            "cars_failed": cars_failed,
            "images_upserted": images_upserted,
            "protection_events": 0,
            "failures": failures,
            "all_listing_origin_urls": all_listing_origin_urls,
            "all_listing_origin_ids": all_listing_origin_ids,
        }

    def _sync_listing_streaming_browser_fallback(
        self,
        *,
        make: str | None,
        model: str | None,
        lane: str,
        limit: int | None,
        effective_only_new: bool,
        listing_url: str | None = None,
        year_min: int | None = None,
        year_max: int | None = None,
    ) -> dict[str, Any]:
        batch_size = self.settings.celery.batch_size
        all_raw_urls: list[str] = []
        seen_urls: set[str] = set()
        pending_urls: list[str] = []
        pages_info: list[dict[str, Any]] = []
        skipped_existing = 0
        cars_upserted = 0
        cars_failed = 0
        images_upserted = 0
        failures: list[dict[str, str]] = []
        page = None
        applied_filters: dict[str, str | int | None] = {
            "make": make,
            "model": model,
            "year_min": year_min,
            "year_max": year_max,
        }
        listing_items_count = 0

        def _flush_pending() -> None:
            nonlocal cars_upserted, cars_failed, images_upserted, failures, pending_urls
            while len(pending_urls) >= batch_size:
                chunk = pending_urls[:batch_size]
                pending_urls = pending_urls[batch_size:]
                try:
                    batch_result = self.sync_batch(chunk, lane=lane)
                    cars_upserted += int(batch_result.get("cars_upserted", 0))
                    cars_failed += int(batch_result.get("cars_failed", 0))
                    images_upserted += int(batch_result.get("images_upserted", 0))
                    failures.extend(batch_result.get("failures", []))
                except Exception as exc:
                    cars_failed += len(chunk)
                    failures.append({"vehicle_url": "browser_fallback_batch", "error": str(exc)})

        try:
            page, applied_filters = self._open_listing_for_stream(
                make=make,
                model=model,
                listing_url=listing_url,
                year_min=year_min,
                year_max=year_max,
            )

            listing_items = self._extract_listing_items_with_wait(page)
            listing_result = self._build_sync_result_from_listing_items(
                listing_items=listing_items,
                make=make,
                model=model,
                lane=lane,
                limit=limit,
                effective_only_new=effective_only_new,
                applied_filters=applied_filters,
                listing_url=listing_url,
            )
            if listing_result is not None:
                listing_items_count = int(listing_result["listing"]["vehicles_collected"])
                return listing_result

            for page_number in range(1, self.settings.listing.max_pages_per_run + 1):
                page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
                pages_info.append({"page_number": page_number, "links_found": len(page_result.vehicle_links)})
                page_urls = self._extract_page_urls(page_result, all_raw_urls, seen_urls)
                for discovered in page_urls:
                    if discovered not in all_raw_urls:
                        all_raw_urls.append(discovered)

                if not page_urls:
                    page_result, page_urls = self._recover_empty_listing_page(
                        page,
                        page_number=page_number,
                        all_raw_urls=all_raw_urls,
                        seen_urls=seen_urls,
                    )
                    pages_info[-1]["links_found"] = len(page_result.vehicle_links)
                    if page_number == 1:
                        listing_result = self._build_sync_result_from_listing_items(
                            listing_items=self._extract_listing_items_with_wait(page),
                            make=make,
                            model=model,
                            lane=lane,
                            limit=limit,
                            effective_only_new=effective_only_new,
                            applied_filters=applied_filters,
                            listing_url=listing_url,
                        )
                        if listing_result is not None:
                            listing_items_count = int(listing_result["listing"]["vehicles_collected"])
                            return listing_result
                    for discovered in page_urls:
                        if discovered not in all_raw_urls:
                            all_raw_urls.append(discovered)
                    if not page_urls:
                        break

                fresh_urls = page_urls
                if effective_only_new:
                    fresh_urls, page_skipped = self._filter_known_urls(page_urls)
                    skipped_existing += page_skipped

                if limit is not None and limit > 0:
                    remaining = max(0, limit - len(pending_urls) - (cars_upserted + cars_failed))
                    if remaining <= 0:
                        break
                    fresh_urls = fresh_urls[:remaining]

                pending_urls.extend(fresh_urls)
                _flush_pending()

                if limit is not None and limit > 0 and (cars_upserted + cars_failed + len(pending_urls)) >= limit:
                    break
                if not page_result.next_page_detected:
                    break
                if self.settings.listing.collect_current_page_only:
                    break

                if not self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1):
                    break

            if pending_urls:
                try:
                    batch_result = self.sync_batch(pending_urls, lane=lane)
                    cars_upserted += int(batch_result.get("cars_upserted", 0))
                    cars_failed += int(batch_result.get("cars_failed", 0))
                    images_upserted += int(batch_result.get("images_upserted", 0))
                    failures.extend(batch_result.get("failures", []))
                except Exception as exc:
                    cars_failed += len(pending_urls)
                    failures.append({"vehicle_url": "browser_fallback_final", "error": str(exc)})
        finally:
            if page is not None:
                page.close()

        all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url}
        all_listing_origin_ids = {
            origin_id
            for origin_id in (self._extract_db_origin_id_from_url(url) for url in all_raw_urls)
            if origin_id
        }
        total = len(all_raw_urls) if not effective_only_new else max(0, len(all_raw_urls) - skipped_existing)
        return {
            "listing": {
                "status": "ok",
                "listing_url": listing_url or self.settings.listing.cars_url,
                "applied_filters": applied_filters,
                "pages_collected": len(pages_info),
                "vehicles_collected": len(all_raw_urls),
                "vehicle_urls": all_raw_urls,
                "early_stopped": False,
                "truncated_by_time_budget": False,
                "pages": pages_info,
            },
            "total": total,
            "skipped_existing": skipped_existing,
            "cars_upserted": cars_upserted,
            "cars_failed": cars_failed,
            "images_upserted": images_upserted,
            "protection_events": 0,
            "failures": failures,
            "all_listing_origin_urls": all_listing_origin_urls,
            "all_listing_origin_ids": all_listing_origin_ids,
        }

    def _get_page(self) -> Page:
        if not self.context:
            self._new_context()
        return self.context.new_page()

    @retryable(max_attempts=3, jitter_seconds=0.25)
    def scrape_vehicle_detail(self, vehicle_url: str):
        page = self._get_page()
        try:
            return self._scrape_on_page(page, vehicle_url)
        finally:
            page.close()

    _JS_EXTRACT = """
    () => {
        try {
            const nextData = window.__NEXT_DATA__;
            const listing = nextData?.props?.pageProps?.reduxWrapperActionsGIPP
                ?.find((entry) => entry && entry.payload && entry.payload.listing)
                ?.payload?.listing;
            if (listing && listing.details) {
                const detailSections = listing.details;
                const normalizeSection = (items) => Array.isArray(items)
                    ? items.map((item) => ({
                        label: item?.label || '',
                        value: item?.value ?? '',
                        slug: item?.slug || '',
                    }))
                    : [];
                return {
                    ok: true,
                    source: 'next_data',
                    listing: {
                        name: listing.name || '',
                        description: listing.description || listing.long_description || '',
                        absolute_url: listing.absolute_url || {},
                        short_url: listing.short_url || '',
                        price: listing.price || {},
                        location: listing.location || {},
                        posted_timestamp: listing.posted_timestamp || null,
                        tracking: listing.tracking || {},
                        categories: Array.isArray(listing.categories) ? listing.categories : [],
                        details: {
                            make_model_trim: normalizeSection(detailSections.make_model_trim),
                            primary: normalizeSection(detailSections.primary),
                            secondary: normalizeSection(detailSections.secondary),
                            rental_details: normalizeSection(detailSections.rental_details),
                            requirements: normalizeSection(detailSections.requirements),
                        },
                        photos: Array.isArray(listing.photos_combined)
                            ? listing.photos_combined.map((photo) => photo?.url || photo?.large || photo?.medium || photo?.small || '').filter(Boolean)
                            : Array.isArray(listing.photos)
                                ? listing.photos.map((photo) => photo?.url || photo?.large || photo?.medium || photo?.small || '').filter(Boolean)
                                : [],
                    },
                };
            }

            const scripts = document.querySelectorAll('script:not([src])');
            for (const s of scripts) {
                const t = s.textContent || '';
                if (!t.includes('inventoryView') || !t.includes('attributes')) continue;
                const start = t.indexOf('{');
                if (start < 0) continue;
                let depth = 0, end = -1;
                for (let i = start; i < t.length; i++) {
                    if (t[i] === '{') depth++;
                    else if (t[i] === '}') { depth--; if (depth === 0) { end = i; break; } }
                }
                if (end < 0) continue;
                try {
                    const obj = JSON.parse(t.slice(start, end + 1));
                    const iv = obj.inventoryView;
                    if (!iv || !iv.attributes) continue;
                    const attr = iv.attributes;
                    const imgs = (iv.imageDimensions && iv.imageDimensions.keys && iv.imageDimensions.keys.$values)
                        ? iv.imageDimensions.keys.$values : [];
                    const bid = (obj.auctionInformation && obj.auctionInformation.biddingInformation)
                        ? obj.auctionInformation.biddingInformation : {};
                    const prebid = (obj.auctionInformation && obj.auctionInformation.prebidInformation)
                        ? obj.auctionInformation.prebidInformation : {};
                    return {
                        ok: true,
                        SalvageId: attr.SalvageId || '',
                        StockNumber: attr.StockNumber || '',
                        Year: attr.Year || '',
                        Make: attr.Make || '',
                        Model: attr.Model || '',
                        Series: attr.Series || '',
                        BodyStyleName: attr.BodyStyleName || '',
                        Cylinders: attr.Cylinders || '',
                        DriveLineTypeDesc: attr.DriveLineTypeDesc || '',
                        EngineSize: (attr.EngineInformation || attr.EngineSize || '').trim(),
                        FuelTypeCode: attr.FuelTypeCode || '',
                        Transmission: attr.Transmission || '',
                        ExteriorColor: attr.ExteriorColor || '',
                        PrimaryDamageDesc: attr.PrimaryDamageDesc || '',
                        SecondaryDamageDesc: attr.SecondaryDamageDesc || '',
                        ODOValue: attr.ODOValue || '',
                        ODOBrand: attr.ODOBrand || '',
                        RunAndDrive: attr.RunAndDrive || '',
                        Keys: attr.Keys || '',
                        BranchName: attr.BranchName || '',
                        AuctionDateTime: attr.AuctionDateTime || '',
                        Title: attr.Title || '',
                        TitleBrand: attr.TitleBrand || '',
                        TitleCode: attr.TitleCode || '',
                        EstRepairCost: attr.EstRepairCost || '',
                        VehicleGrade: attr.VehicleGrade || '',
                        highBidAmount: prebid.highBidAmount || bid.highBidAmount || '',
                        buyNowPrice: prebid.buyNowPrice || bid.buyNowPrice || '',
                        acv: attr.ProviderACV || '',
                        BranchNumber: attr.BranchNumber || '',
                        imageKeys: imgs.map(i => i.k || '').filter(Boolean),
                    };
                } catch (_) { continue; }
            }
        } catch (_) {}
        return { ok: false };
    }
    """

    @staticmethod
    def _build_car_from_js(js_data: dict, vehicle_url: str) -> dict:
        if not js_data or not js_data.get("ok"):
            return {}

        if js_data.get("source") == "next_data":
            listing = js_data.get("listing") or {}
            details = listing.get("details") or {}
            categories = listing.get("categories") or []
            absolute_url = listing.get("absolute_url") or {}
            location = listing.get("location") or {}
            tracking = listing.get("tracking") or {}

            def detail_value_by_slug(target_slug: str) -> Any:
                target = target_slug.strip().lower()
                for section_items in details.values():
                    if not isinstance(section_items, list):
                        continue
                    for item in section_items:
                        if not isinstance(item, Mapping):
                            continue
                        if str(item.get("slug") or "").strip().lower() == target:
                            return item.get("value")
                return None

            def pick_photo_url(photo: Any) -> str:
                if isinstance(photo, str):
                    return photo.strip()
                if not isinstance(photo, Mapping):
                    return ""
                for key in ("url", "main", "large", "medium", "small", "micro"):
                    value = photo.get(key)
                    if value:
                        return str(value).strip()
                return ""

            detail_sections: dict[str, list[dict[str, Any]]] = {}
            flat_details: dict[str, dict[str, dict[str, Any]]] = {}
            for section_name, items in details.items():
                if not isinstance(items, list):
                    continue
                normalized_items: list[dict[str, Any]] = []
                for item in items:
                    if not isinstance(item, Mapping):
                        continue
                    label = str(item.get("label") or "").strip()
                    value = item.get("value")
                    slug = str(item.get("slug") or "").strip()
                    normalized_items.append({"label": {"en": label}, "value": {"en": value}, "slug": slug})
                    if label:
                        flat_details[label] = {"en": {"label": label, "value": value}}
                if normalized_items:
                    detail_sections[section_name] = normalized_items

            category_v2 = None
            category_make = None
            category_model = None
            if categories:
                names_en = [str(item.get("name") or "").strip() for item in categories if isinstance(item, Mapping)]
                slug_paths = [str(item.get("full_slug") or item.get("slug") or "").strip() for item in categories if isinstance(item, Mapping)]
                ids = [item.get("legacy_id") for item in categories if isinstance(item, Mapping)]
                category_v2 = {
                    "names_en": names_en,
                    "slug_paths": slug_paths,
                    "ids": ids,
                }
                if len(names_en) >= 4:
                    category_make = names_en[2] or None
                    category_model = names_en[3] or None

            photos: list[str] = []
            for photo in listing.get("photos_combined") or []:
                picked = pick_photo_url(photo)
                if picked and picked not in photos:
                    photos.append(picked)
            for photo in listing.get("photos") or []:
                picked = pick_photo_url(photo)
                if picked and picked not in photos:
                    photos.append(picked)

            posted_timestamp = listing.get("posted_timestamp")
            posted_at_iso = None
            if isinstance(posted_timestamp, (int, float)) and posted_timestamp > 0:
                posted_at_iso = datetime.fromtimestamp(posted_timestamp, tz=timezone.utc).isoformat()

            price = listing.get("price") if isinstance(listing.get("price"), Mapping) else {}
            price_raw = price.get("raw") or price.get("formatted")
            neighborhood_name = tracking.get("neighbourhood", {}).get("name") if isinstance(tracking.get("neighbourhood"), Mapping) else None
            year = detail_value_by_slug("year")
            body_type = detail_value_by_slug("body_type")
            kilometers = detail_value_by_slug("kilometers")
            engine_capacity = detail_value_by_slug("engine_capacity_cc")
            transmission_type = detail_value_by_slug("transmission_type")
            steering_side = detail_value_by_slug("steering_side")
            exterior_color = detail_value_by_slug("exterior_color")
            seller_type = detail_value_by_slug("seller_type")
            trim = detail_value_by_slug("motors_trim")

            summary: dict[str, Any] = {
                "source_url": vehicle_url,
                "name": {"en": listing.get("name")},
                "title": listing.get("name"),
                "description": listing.get("description") or listing.get("long_description"),
                "make": category_make,
                "model": category_model,
                "trim": trim,
                "year": year,
                "body_type": body_type,
                "odometer": kilometers,
                "kilometers": kilometers,
                "engine": engine_capacity,
                "engine_volume": engine_capacity,
                "transmission_type": transmission_type,
                "gearbox": transmission_type,
                "steering_side": steering_side,
                "steering_wheel": steering_side,
                "exterior_color": exterior_color,
                "color": exterior_color,
                "seller": seller_type,
                "price": price_raw,
                "buy_now": price_raw,
                "currency": price.get("currency") or "AED",
                "details_v2": detail_sections,
                "details": flat_details,
                "image_urls": photos,
                "photo_mains": photos,
                "category_v2": category_v2,
                "location": location.get("name"),
                "location_name": location.get("name"),
                "site": {"en": "UAE"},
                "posted_at": posted_at_iso,
                "permalink": listing.get("short_url") or vehicle_url,
                "absolute_url": absolute_url,
                "id": tracking.get("legacy_id") or listing.get("listing_id") or listing.get("object_id"),
                "objectID": listing.get("encoded_object_id") or listing.get("object_id"),
                "uuid": listing.get("listing_uuid") or listing.get("uuid"),
            }
            if neighborhood_name:
                summary["neighbourhood"] = {"en": neighborhood_name}
            return summary

        brnch = str(js_data.get("BranchNumber", "") or "").strip()
        img_keys = js_data.get("imageKeys") or []
        image_urls = [
            f"https://vis.dubizzle.com/resizer?imageKeys={k}&width=845&height=633"
            for k in img_keys
        ]

        return {
            "source_url": vehicle_url,
            "lot_number": js_data.get("StockNumber") or js_data.get("SalvageId"),
            "year": js_data.get("Year"),
            "make": js_data.get("Make"),
            "model": js_data.get("Model"),
            "trim": js_data.get("Series"),
            "body_type": js_data.get("BodyStyleName"),
            "cylinders": js_data.get("Cylinders"),
            "drive": js_data.get("DriveLineTypeDesc"),
            "engine": js_data.get("EngineSize"),
            "fuel_type": js_data.get("FuelTypeCode"),
            "gearbox": js_data.get("Transmission"),
            "color": js_data.get("ExteriorColor"),
            "primary_damage": js_data.get("PrimaryDamageDesc"),
            "secondary_damage": js_data.get("SecondaryDamageDesc"),
            "odometer": js_data.get("ODOValue"),
            "run_and_drive": js_data.get("RunAndDrive"),
            "keys": js_data.get("Keys"),
            "location": js_data.get("BranchName"),
            "auction_date": js_data.get("AuctionDateTime"),
            "title": js_data.get("Title"),
            "current_bid": js_data.get("highBidAmount"),
            "buy_now": js_data.get("buyNowPrice"),
            "actual_cash_value": js_data.get("acv"),
            "estimated_repair_cost": js_data.get("EstRepairCost"),
            "image_urls": image_urls,
        }

    @staticmethod
    def _build_payload_insights(vehicle_summary: dict) -> dict:
        currency = vehicle_summary.get("currency") or "USD"
        effective_price = vehicle_summary.get("buy_now") or vehicle_summary.get("price")
        return {
            "vehicle_core": vehicle_summary,
            "pricing": {
                "buy_now": effective_price,
                "current_bid": vehicle_summary.get("current_bid"),
                "actual_cash_value": vehicle_summary.get("actual_cash_value"),
                "estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"),
                "currency": currency,
            },
            "bids": {"amount": vehicle_summary.get("current_bid"), "currency": currency},
            "damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")},
            "auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")},
            "images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []},
        }

    def _scrape_on_page(self, page: Page, vehicle_url: str):
        trace_id = self._new_trace_id("scrape")
        started_at = time.perf_counter()

        if self.settings.block_resources:
            BrowserFactory.enable_resource_blocking(page)

        capture = NetworkCapture(self.settings)
        capture.attach(page, origin_url=vehicle_url)

        page.goto(vehicle_url, wait_until="commit", timeout=60_000)

        try:
            page.wait_for_load_state("domcontentloaded", timeout=5_000)
        except PlaywrightTimeoutError:
            pass

        js_data: dict = {}
        try:
            js_data = page.evaluate(self._JS_EXTRACT) or {}
        except Exception:
            pass

        if js_data.get("ok"):
            vehicle_summary = self._build_car_from_js(js_data, vehicle_url)
            has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number"))
            if not has_identity:
                raise SiteStructureChangedError(f"JS extraction returned no vehicle identity for {vehicle_url}")

            db_record = self.car_mapper.map_to_car_record(
                vehicle_url=vehicle_url,
                vehicle_summary=vehicle_summary,
                payload_insights=self._build_payload_insights(vehicle_summary),
            )
            network_dump = capture.export()
            result = {
                "trace_id": trace_id,
                "source_url": vehicle_url,
                "fetched_at_epoch": int(time.time()),
                "elapsed_seconds": round(time.perf_counter() - started_at, 3),
                "network": network_dump,
                "vehicle_summary": vehicle_summary,
                "payload_insights": {},
                "embedded_json": [],
                "dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
                "access_notes": {},
                "db_record": db_record.model_dump(mode="json"),
            }
            if self.settings.raw_output_json:
                save_to_json(network_dump, self.settings.raw_output_json)
            return result

        # Резервный путь: полный HTML-парсинг.
        try:
            page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=400)
        except PlaywrightTimeoutError:
            pass

        html = page.content()
        try:
            dom_text = page.evaluate("() => document.body?.textContent || ''")
        except Exception:
            dom_text = ""
        network_dump = capture.export()
        parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump)
        self._raise_if_blocked_or_incomplete(parsed, vehicle_url)

        db_record = self.car_mapper.map_to_car_record(
            vehicle_url=vehicle_url,
            vehicle_summary=parsed.get("vehicle_summary", {}),
            payload_insights=parsed.get("payload_insights", {}),
        )

        result = {
            "trace_id": trace_id,
            "source_url": vehicle_url,
            "fetched_at_epoch": int(time.time()),
            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
            "network": network_dump,
            **parsed,
            "db_record": db_record.model_dump(mode="json"),
        }

        if self.settings.raw_output_json:
            save_to_json(network_dump, self.settings.raw_output_json)
        return result

    @staticmethod
    def _extract_dubizzle_json_from_html(html: str, vehicle_url: str) -> dict | None:
        """Извлекает DUBIZZLE inventoryView.attributes из HTML без браузера.

        Использует json.JSONDecoder.raw_decode для быстрого поиска JSON
        вместо посимвольного сканирования скобок.
        """
        next_match = NEXT_DATA_RE.search(html)
        if next_match:
            try:
                next_data = json.loads(html_module.unescape(next_match.group(1)))
                actions = (
                    next_data.get("props", {})
                    .get("pageProps", {})
                    .get("reduxWrapperActionsGIPP", [])
                )
                for entry in actions:
                    payload = entry.get("payload") if isinstance(entry, dict) else None
                    listing = payload.get("listing") if isinstance(payload, dict) else None
                    if isinstance(listing, dict) and isinstance(listing.get("details"), dict):
                        return {
                            "ok": True,
                            "source": "next_data",
                            "listing": listing,
                        }
            except Exception:
                pass

        search = "inventoryView"
        pos = html.find(search)
        if pos < 0:
            return None

        script_start = html.rfind("<script", 0, pos)
        if script_start < 0:
            return None
        tag_end = html.find(">", script_start)
        if tag_end < 0:
            return None
        content_start = html.find("{", tag_end)
        if content_start < 0:
            return None

        decoder = json.JSONDecoder()
        try:
            obj, _ = decoder.raw_decode(html, content_start)
        except (json.JSONDecodeError, ValueError):
            return None

        iv = obj.get("inventoryView")
        if not iv or not iv.get("attributes"):
            return None

        attr = iv["attributes"]
        imgs = []
        try:
            imgs = iv.get("imageDimensions", {}).get("keys", {}).get("$values", []) or []
        except Exception:
            pass

        bid = {}
        prebid = {}
        try:
            ai = obj.get("auctionInformation", {})
            bid = ai.get("biddingInformation", {}) or {}
            prebid = ai.get("prebidInformation", {}) or {}
        except Exception:
            pass

        img_keys = [i.get("k", "") for i in imgs if i.get("k")]
        return {
            "ok": True,
            "SalvageId": attr.get("SalvageId", ""),
            "StockNumber": attr.get("StockNumber", ""),
            "Year": attr.get("Year", ""),
            "Make": attr.get("Make", ""),
            "Model": attr.get("Model", ""),
            "Series": attr.get("Series", ""),
            "BodyStyleName": attr.get("BodyStyleName", ""),
            "Cylinders": attr.get("Cylinders", ""),
            "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""),
            "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(),
            "FuelTypeCode": attr.get("FuelTypeCode", ""),
            "Transmission": attr.get("Transmission", ""),
            "ExteriorColor": attr.get("ExteriorColor", ""),
            "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""),
            "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""),
            "ODOValue": attr.get("ODOValue", ""),
            "ODOBrand": attr.get("ODOBrand", ""),
            "RunAndDrive": attr.get("RunAndDrive", ""),
            "Keys": attr.get("Keys", ""),
            "BranchName": attr.get("BranchName", ""),
            "AuctionDateTime": attr.get("AuctionDateTime", ""),
            "Title": attr.get("Title", ""),
            "TitleBrand": attr.get("TitleBrand", ""),
            "TitleCode": attr.get("TitleCode", ""),
            "EstRepairCost": attr.get("EstRepairCost", ""),
            "VehicleGrade": attr.get("VehicleGrade", ""),
            "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""),
            "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""),
            "acv": attr.get("ProviderACV", ""),
            "BranchNumber": attr.get("BranchNumber", ""),
            "imageKeys": img_keys,
        }

    def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord:
        if not self.context:
            self._new_context()
        assert self.context is not None
        last_error: Exception | None = None
        max_attempts = max(1, self.settings.fast_path_max_attempts)
        timeout_ms = max(1000, self.settings.fast_path_timeout_ms)
        for attempt in range(1, max_attempts + 1):
            try:
                response = self.context.request.get(vehicle_url, timeout=timeout_ms)
                if not response.ok:
                    raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}")

                html = response.text()

                js_data = self._extract_dubizzle_json_from_html(html, vehicle_url)
                if not js_data:
                    raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}")

                vehicle_summary = self._build_car_from_js(js_data, vehicle_url)
                has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number"))
                if not has_identity:
                    raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}")

                db_record = self.car_mapper.map_to_car_record(
                    vehicle_url=vehicle_url,
                    vehicle_summary=vehicle_summary,
                    payload_insights=self._build_payload_insights(vehicle_summary),
                )
                return CarRecord.model_validate(db_record.model_dump(mode="json"))
            except Exception as exc:
                last_error = exc
                if attempt < max_attempts:
                    time.sleep(0.2)

        if last_error is not None:
            raise last_error
        raise RuntimeError(f"HTTP fast-path failed for {vehicle_url}")

    def _cookie_header_for_context(self) -> str:
        if not self.context:
            return ""
        try:
            cookies = self.context.cookies()
        except Exception:
            return ""
        pairs = [
            f"{item.get('name')}={item.get('value')}"
            for item in cookies
            if item.get("name") and item.get("value") is not None
        ]
        return "; ".join(pairs)

    def _scrape_via_raw_http(
        self,
        vehicle_url: str,
        *,
        cookie_header: str,
        user_agent: str,
    ) -> CarRecord:
        """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive)."""
        headers = {
            "User-Agent": user_agent,
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.9",
            "Cache-Control": "no-cache",
            "Pragma": "no-cache",
            "Connection": "keep-alive",
            "Upgrade-Insecure-Requests": "1",
        }
        if cookie_header:
            headers["Cookie"] = cookie_header

        response = self._http_pool.request("GET", vehicle_url, headers=headers)
        if response.status >= 400:
            raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}")
        html = response.data.decode("utf-8", errors="ignore")

        js_data = self._extract_dubizzle_json_from_html(html, vehicle_url)
        if not js_data:
            raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}")

        vehicle_summary = self._build_car_from_js(js_data, vehicle_url)
        has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number"))
        if not has_identity:
            raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}")

        db_record = self.car_mapper.map_to_car_record(
            vehicle_url=vehicle_url,
            vehicle_summary=vehicle_summary,
            payload_insights=self._build_payload_insights(vehicle_summary),
        )
        return CarRecord.model_validate(db_record.model_dump(mode="json"))

    def sync_vehicle(self, vehicle_url: str, lane: str = "dubizzle"):
        trace_id = self._new_trace_id("sync-vehicle")
        started_at = time.perf_counter()
        self.persistence.create_tables()
        run_id = self.persistence.start_sync_run(lane=lane)
        ids_fetched = 1
        cars_upserted = 0
        cars_failed = 0
        images_upserted = 0
        status = "failed"
        error_summary = None
        try:
            scrape_result = self.scrape_vehicle_detail(vehicle_url)
            db_record = scrape_result.get("db_record")
            if not db_record:
                raise RuntimeError("Scrape result does not contain db_record")
            record = CarRecord.model_validate(db_record)
            upsert = self.persistence.upsert_car(record)
            cars_upserted = 1
            images_upserted = int(upsert.get("images_upserted", 0))
            status = "success"
            return {
                "trace_id": trace_id,
                "status": status,
                "run_id": run_id,
                "vehicle_url": vehicle_url,
                "db_action": upsert.get("action"),
                "images_upserted": images_upserted,
                "elapsed_seconds": round(time.perf_counter() - started_at, 3),
                "db_record": db_record,
            }
        except Exception as exc:
            cars_failed = 1
            status = "failed"
            error_summary = str(exc)
            raise
        finally:
            self.persistence.finish_sync_run(
                run_id,
                status=status,
                ids_fetched=ids_fetched,
                cars_upserted=cars_upserted,
                cars_failed=cars_failed,
                images_upserted=images_upserted,
                error_summary=error_summary,
            )

    # ── Настройки sync_batch ──
    _HTTP_MICRO_BATCH = 25          # URL за микро-батч
    _HTTP_MAX_RETRIES = 2           # Повторов на URL до перехода в браузер
    _HTTP_RETRY_DELAYS = (0.4, 1.0) # Задержки между повторами
    _FALLBACK_PARALLEL_PAGES = 4    # Параллельных вкладок для fallback

    def _browser_fallback_parallel(
        self,
        fallback_urls: list[tuple[str, int]],
        total: int,
        n_pages: int,
    ) -> dict:
        records: list[CarRecord] = []
        failures: list[dict[str, str]] = []
        cars_failed = 0
        protection_events = 0

        slices: list[list[tuple[str, int]]] = [[] for _ in range(n_pages)]
        for i, item in enumerate(fallback_urls):
            slices[i % n_pages].append(item)

        def _process_slice(url_slice: list[tuple[str, int]]) -> dict:
            local_records: list[CarRecord] = []
            local_failures: list[dict[str, str]] = []
            local_failed = 0
            local_protection = 0
            page: Page | None = None
            processed_local = 0
            try:
                for url, global_idx in url_slice:
                    processed_local += 1
                    # Heartbeat в Redis progress: даже если anti-bot тормозит fallback,
                    # watchdog видит живую задачу и не убивает её как stalled.
                    self._report_progress(
                        "browser_fallback_progress",
                        fallback_processed=processed_local,
                        fallback_total=len(url_slice),
                        vehicle_index=global_idx,
                        vehicle_total=total,
                    )

                    # Если anti-bot уже активен (много fallback URL), снижаем burst-нагрузку.
                    if len(url_slice) >= 20:
                        time.sleep(random.uniform(0.2, 0.7))

                    if page is None:
                        page = self._get_page()
                        if self.settings.block_resources:
                            BrowserFactory.enable_resource_blocking(page)

                    try:
                        page.goto(
                            url,
                            wait_until="commit",
                            timeout=max(3_000, int(self.settings.fallback_navigation_timeout_ms)),
                        )
                    except Exception as exc:
                        if self._is_protection_or_network_error(exc):
                            local_protection += 1
                        local_failed += 1
                        local_failures.append({"vehicle_url": url, "error": str(exc)})
                        logger.error("[%d/%d] Failed to open %s: %s", global_idx, total, url, exc)
                        try:
                            page.close()
                        except Exception:
                            pass
                        page = None
                        continue

                    try:
                        js_data: dict = {}
                        try:
                            js_data = page.evaluate(self._JS_EXTRACT) or {}
                        except Exception:
                            pass

                        if not js_data.get("ok"):
                            try:
                                page.wait_for_load_state("domcontentloaded", timeout=3_000)
                            except PlaywrightTimeoutError:
                                pass
                            try:
                                js_data = page.evaluate(self._JS_EXTRACT) or {}
                            except Exception:
                                pass

                        if js_data.get("ok"):
                            vehicle_summary = self._build_car_from_js(js_data, url)
                            has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number"))
                            if not has_identity:
                                raise SiteStructureChangedError(
                                    f"JS extraction returned no vehicle identity for {url}"
                                )
                            db_record = self.car_mapper.map_to_car_record(
                                vehicle_url=url,
                                vehicle_summary=vehicle_summary,
                                payload_insights=self._build_payload_insights(vehicle_summary),
                            )
                        else:
                            try:
                                page.wait_for_selector(
                                    "#VehicleDetailViewModel, .veh-details, .vehicle-details, "
                                    "[data-uname='vehicleDetailPage']",
                                    timeout=400,
                                )
                            except PlaywrightTimeoutError:
                                pass
                            page_html = page.content()
                            try:
                                dom_text = page.evaluate("() => document.body?.textContent || ''")
                            except Exception:
                                dom_text = ""
                            network_dump = {
                                "requests": [],
                                "json_responses": [],
                                "capture_limits": {},
                            }
                            parsed = self.vehicle_parser.normalize(url, page_html, dom_text, network_dump)
                            self._raise_if_blocked_or_incomplete(parsed, url)
                            db_record = self.car_mapper.map_to_car_record(
                                vehicle_url=url,
                                vehicle_summary=parsed.get("vehicle_summary", {}),
                                payload_insights=parsed.get("payload_insights", {}),
                            )

                        record = CarRecord.model_validate(db_record.model_dump(mode="json"))
                        local_records.append(record)
                        logger.debug(
                            "[%d/%d] Parsed %s %s %s (fallback)",
                            global_idx, total, record.brand, record.model, record.year or "?",
                        )
                    except Exception as exc:
                        if self._is_protection_or_network_error(exc):
                            local_protection += 1
                        local_failed += 1
                        local_failures.append({"vehicle_url": url, "error": str(exc)})
                        logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc)
            finally:
                if page is not None:
                    try:
                        page.close()
                    except Exception:
                        pass
            return {
                "records": local_records,
                "failures": local_failures,
                "cars_failed": local_failed,
                "protection_events": local_protection,
            }

        # Одна страница — в главном потоке.
        if n_pages == 1:
            res = _process_slice(slices[0] if slices else [])
            records.extend(res["records"])
            failures.extend(res["failures"])
            cars_failed += res["cars_failed"]
            protection_events += res["protection_events"]
        else:
            # Несколько страниц — параллельно, каждый поток со своей Page.
            # Таймаут 5 минут на весь fallback — если страницы зависли, не блокируем навсегда.
            _fallback_timeout = max(300, len(fallback_urls) * 30)
            with ThreadPoolExecutor(max_workers=n_pages) as executor:
                futures = [executor.submit(_process_slice, s) for s in slices if s]
                for fut in as_completed(futures, timeout=_fallback_timeout):
                    try:
                        res = fut.result(timeout=60)
                    except FuturesTimeoutError:
                        logger.error("Browser fallback thread timed out")
                        cars_failed += 1
                        continue
                    records.extend(res["records"])
                    failures.extend(res["failures"])
                    cars_failed += res["cars_failed"]
                    protection_events += res["protection_events"]

        return {
            "records": records,
            "failures": failures,
            "cars_failed": cars_failed,
            "protection_events": protection_events,
        }

    def sync_batch(
        self,
        vehicle_urls: list[str],
        lane: str = "dubizzle_cars",
        parallel_tabs: int | None = None,
    ) -> dict:
        # Runtime config может меняться на лету (добавили/убрали бренды, диапазоны и т.п.).
        # Подхватываем обновление перед каждым batch, чтобы фильтрация применялась сразу.
        self._reload_runtime_config()

        trace_id = self._new_trace_id("sync-batch")
        started_at = time.perf_counter()
        num_workers = parallel_tabs or self.settings.parallel_tabs
        num_workers = min(num_workers, len(vehicle_urls), 48)

        cars_upserted = 0
        cars_failed = 0
        cars_filtered = 0
        images_upserted = 0
        records: list[CarRecord] = []
        failures: list[dict[str, str]] = []
        http_successes = 0
        http_fallbacks = 0
        protection_events = 0

        total = len(vehicle_urls)
        logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers)
        self._report_progress(
            "sync_batch_started",
            batch_total=total,
            batch_workers=num_workers,
            http_ok=0,
            http_fallback=0,
            batch_records_ready=0,
            batch_failures=0,
        )

        cookie_header = self._cookie_header_for_context()
        user_agent = (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
            "Gecko/20100101 Firefox/128.0"
        )

        # ── Фаза 1: HTTP fast-path ──
        fallback_urls: list[tuple[str, int]] = []
        processed_indices: set[int] = set()
        fallback_indices: set[int] = set()

        def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]:
            idx, url = idx_url
            # Небольшой анти-бёрст джиттер: разносим старт запросов в пуле,
            # чтобы N воркеров не били в одну TLS-/PX-волну.
            time.sleep(random.uniform(0.0, 0.15))
            last_exc: Exception | None = None
            for attempt in range(1 + self._HTTP_MAX_RETRIES):
                try:
                    return idx, self._scrape_via_raw_http(
                        url,
                        cookie_header=cookie_header,
                        user_agent=user_agent,
                    )
                except Exception as exc:
                    last_exc = exc
                    if attempt < self._HTTP_MAX_RETRIES:
                        time.sleep(self._HTTP_RETRY_DELAYS[min(attempt, len(self._HTTP_RETRY_DELAYS) - 1)])
            return idx, last_exc  # type: ignore[return-value]

        indexed_urls = list(enumerate(vehicle_urls))
        # Таймаут на весь HTTP-пул: (connect 5 + read 20) × retries × URLs / workers + запас.
        # Если пул зависнет дольше — отпускаем зависшие потоки и уходим в fallback.
        _per_url_budget = (5 + 20) * (1 + self._HTTP_MAX_RETRIES) + 5
        _http_phase_timeout = max(120, _per_url_budget * max(1, total // max(1, num_workers)))
        http_processed = 0
        with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor:
            try:
                for idx, result in executor.map(
                    _fetch_one_with_retry, indexed_urls, timeout=_http_phase_timeout,
                ):
                    processed_indices.add(idx)
                    http_processed += 1
                    if isinstance(result, Exception):
                        http_fallbacks += 1
                        logger.debug(
                            "[%d/%d] HTTP fast-path failed for %s: %s",
                            idx + 1, total, vehicle_urls[idx], result,
                        )
                        fallback_urls.append((vehicle_urls[idx], idx + 1))
                        fallback_indices.add(idx)
                    else:
                        records.append(result)
                        http_successes += 1
                        logger.debug(
                            "[%d/%d] Parsed %s %s %s (raw HTTP)",
                            idx + 1, total, result.brand, result.model, result.year or "?",
                        )
                    self._report_progress(
                        "sync_batch_http_progress",
                        batch_total=total,
                        http_processed=http_processed,
                        http_ok=http_successes,
                        http_fallback=http_fallbacks,
                        batch_records_ready=len(records),
                        batch_failures=cars_failed + len(failures),
                    )
            except FuturesTimeoutError:
                logger.error(
                    "HTTP phase timed out after %ds; %d/%d processed, rest go to fallback",
                    _http_phase_timeout, http_successes + http_fallbacks, total,
                )
                # Все ещё не обработанные URL → в fallback
                for i, url in enumerate(vehicle_urls):
                    if i in processed_indices or i in fallback_indices:
                        continue
                    if (url, i + 1) not in fallback_urls:
                        fallback_urls.append((url, i + 1))
                        fallback_indices.add(i)
                        http_fallbacks += 1

        logger.info(
            "HTTP phase done: %d OK, %d fallback (%.1fs)",
            http_successes, http_fallbacks, time.perf_counter() - started_at,
        )
        self._report_progress(
            "sync_batch_http_done",
            batch_total=total,
            http_processed=http_processed,
            http_ok=http_successes,
            http_fallback=http_fallbacks,
            batch_records_ready=len(records),
            batch_failures=cars_failed + len(failures),
        )

        # ── Фаза 2: браузерный fallback ──
        if fallback_urls:
            logger.info(
                "Fallback browser mode: %d/%d URLs, single page",
                len(fallback_urls), total,
            )
            self._report_progress(
                "sync_batch_fallback_started",
                batch_total=total,
                http_ok=http_successes,
                http_fallback=http_fallbacks,
                fallback_total=len(fallback_urls),
                batch_records_ready=len(records),
                batch_failures=cars_failed + len(failures),
            )
            fb_results = self._browser_fallback_parallel(fallback_urls, total, 1)
            records.extend(fb_results["records"])
            cars_failed += fb_results["cars_failed"]
            protection_events += fb_results["protection_events"]
            failures.extend(fb_results["failures"])
            self._report_progress(
                "sync_batch_fallback_done",
                batch_total=total,
                http_ok=http_successes,
                http_fallback=http_fallbacks,
                fallback_total=len(fallback_urls),
                batch_records_ready=len(records),
                batch_failures=cars_failed + len(failures),
                protection_events=protection_events,
            )

        # ── Фаза 2.5: пост-фильтр по runtime_config ──
        filters_cfg = self.runtime_config.filters
        if records and not filters_cfg.is_empty():
            pre_filter_count = len(records)
            records = [
                rec for rec in records
                if filters_cfg.matches({
                    "brand": rec.brand,
                    "model": rec.model,
                    "year": rec.year,
                    "body_type": rec.body_type,
                    "color": rec.color,
                    "drive": rec.drive,
                    "gearbox": rec.gearbox,
                    "price": rec.price,
                    "mileage": rec.mileage,
                })
            ]
            cars_filtered = pre_filter_count - len(records)
            if cars_filtered:
                logger.info(
                    "Runtime filter: %d/%d records filtered out before DB upsert",
                    cars_filtered, pre_filter_count,
                )

        # ── Фаза 3: запись в БД ──
        if records:
            seen_origins: set[str] = set()
            unique_records: list[CarRecord] = []
            for rec in records:
                key = rec.origin_id or rec.origin_url
                if key not in seen_origins:
                    seen_origins.add(key)
                    unique_records.append(rec)
            if len(unique_records) < len(records):
                logger.info("Dedup before DB: %d → %d", len(records), len(unique_records))
            records = unique_records

            try:
                self._report_progress(
                    "sync_batch_db_upsert_started",
                    batch_total=total,
                    batch_records_ready=len(records),
                    http_ok=http_successes,
                    http_fallback=http_fallbacks,
                    batch_failures=cars_failed + len(failures),
                )
                _db_start = time.perf_counter()
                batch_result = self.persistence.upsert_cars_batch(records)
                _db_elapsed = time.perf_counter() - _db_start
                cars_upserted = batch_result["inserted"] + batch_result["updated"]
                images_upserted = batch_result["images_upserted"]
                if _db_elapsed > 10:
                    logger.warning("DB upsert slow: %.1fs for %d records", _db_elapsed, len(records))
                self._report_progress(
                    "sync_batch_db_upsert_done",
                    batch_total=total,
                    batch_records_ready=len(records),
                    cars_upserted=cars_upserted,
                    images_upserted=images_upserted,
                    http_ok=http_successes,
                    http_fallback=http_fallbacks,
                    batch_failures=cars_failed + len(failures),
                )
            except Exception as exc:
                logger.error("Batch upsert failed: %s", exc)
                cars_failed += len(records)
                self._report_progress(
                    "sync_batch_db_upsert_failed",
                    batch_total=total,
                    batch_records_ready=len(records),
                    http_ok=http_successes,
                    http_fallback=http_fallbacks,
                    batch_failures=cars_failed + len(failures),
                    error=str(exc),
                )

        status = "success" if not failures else ("partial_success" if cars_upserted else "failed")
        self._report_progress(
            "sync_batch_done",
            batch_total=total,
            cars_upserted=cars_upserted,
            cars_failed=cars_failed,
            cars_filtered=cars_filtered,
            images_upserted=images_upserted,
            http_ok=http_successes,
            http_fallback=http_fallbacks,
            protection_events=protection_events,
            batch_failures=len(failures),
        )
        return {
            "trace_id": trace_id,
            "status": status,
            "cars_upserted": cars_upserted,
            "cars_failed": cars_failed,
            "cars_filtered": cars_filtered,
            "images_upserted": images_upserted,
            "http_successes": http_successes,
            "http_fallbacks": http_fallbacks,
            "protection_events": protection_events,
            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
            "failures": failures,
        }

    def sync_listing(
        self,
        make: str | None = None,
        model: str | None = None,
        lane: str = "dubizzle_cars",
        limit: int | None = None,
        only_new: bool | None = None,
        listing_url: str | None = None,
        year_min: int | None = None,
        year_max: int | None = None,
        skip_mark_sold: bool = False,
    ):
        # Подхватываем актуальный runtime_config на каждый запуск sync,
        # чтобы добавленные/удалённые бренды/модели применялись без рестарта.
        self._reload_runtime_config()

        # runtime_config — дефолты; CLI/API аргументы приоритетнее.
        rc = self.runtime_config.sync
        if limit is None and rc.limit is not None:
            limit = rc.limit
        if only_new is None and rc.only_new is not None:
            only_new = rc.only_new
        if lane == "dubizzle_cars" and rc.lane is not None:
            lane = rc.lane

        trace_id = self._new_trace_id("sync-listing")
        started_at = time.perf_counter()
        self.persistence.create_tables()
        run_id = self.persistence.start_sync_run(lane=lane)
        cars_upserted = 0
        cars_failed = 0
        cars_filtered = 0
        protection_events = 0
        images_upserted = 0
        total = 0
        skipped_existing = 0
        is_partial_scan = True
        failures: list[dict[str, str]] = []
        listing: dict = {}

        try:
            effective_only_new = self.settings.sync_only_new if only_new is None else only_new
            stream_result = self._sync_listing_streaming(
                make=make,
                model=model,
                lane=lane,
                limit=limit,
                effective_only_new=effective_only_new,
                started_at=started_at,
                listing_url=listing_url,
                year_min=year_min,
                year_max=year_max,
            )
            listing = stream_result["listing"]
            total = stream_result["total"]
            skipped_existing = stream_result["skipped_existing"]
            cars_upserted = stream_result["cars_upserted"]
            cars_failed = stream_result["cars_failed"]
            images_upserted = stream_result["images_upserted"]
            protection_events = int(stream_result.get("protection_events", 0))
            failures.extend(stream_result["failures"])
            all_listing_origin_urls = stream_result["all_listing_origin_urls"]
            all_listing_origin_ids = stream_result.get("all_listing_origin_ids", set())

            logger.info("Streaming sync processed %d vehicles", total)

            # Помечаем проданные авто, исчезнувшие из листинга (только при полном скане).
            is_partial_scan = (
                effective_only_new
                or (limit is not None and limit > 0)
                or listing.get("early_stopped", False)
                or listing.get("truncated_by_time_budget", False)
            )
            if skip_mark_sold:
                logger.debug("Skipping mark_sold: caller requested")
            elif all_listing_origin_ids and not is_partial_scan:
                try:
                    sold_count = self.persistence.mark_sold_not_in_listing(all_listing_origin_ids)
                    if sold_count:
                        logger.info("Marked %d cars as sold by origin_id", sold_count)
                except Exception as exc:
                    logger.warning("Failed to mark sold cars by origin_id: %s", exc)
            elif all_listing_origin_urls and not is_partial_scan:
                try:
                    sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls)
                    if sold_count:
                        logger.info("Marked %d cars as sold", sold_count)
                except Exception as exc:
                    logger.warning("Failed to mark sold cars: %s", exc)
            elif is_partial_scan:
                logger.debug("Skipping mark_sold: partial/incremental scan (only_new=%s, limit=%s, early_stopped=%s)",
                             effective_only_new, limit, listing.get("early_stopped", False))
        except Exception as exc:
            if not failures:
                failures.append({"vehicle_url": "collect_listing", "error": str(exc)})
            logger.error("sync_listing failed: %s (partial progress: %d upserted)", exc, cars_upserted)
        finally:
            status = "success" if not failures else ("partial_success" if cars_upserted else "failed")
            error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None
            self.persistence.finish_sync_run(
                run_id,
                status=status,
                ids_fetched=total,
                cars_upserted=cars_upserted,
                cars_failed=cars_failed,
                images_upserted=images_upserted,
                error_summary=error_summary,
            )

        logger.info(
            "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images",
            run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted,
        )
        discovered_total = max(0, int(total))
        fail_ratio = (cars_failed / discovered_total) if discovered_total > 0 else 0.0
        protection_ratio = (protection_events / discovered_total) if discovered_total > 0 else 0.0
        anti_bot_detected = discovered_total > 0 and (
            (protection_events >= 30 and protection_ratio >= 0.10)
            or fail_ratio >= 0.30
        )
        return {
            "trace_id": trace_id,
            "status": status,
            "run_id": run_id,
            # partial_success допустим — отдельные машины могли не спарситься,
            # это не повод повторять весь bootstrap.
            "full_scan_completed": (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected,
            "only_new_effective": effective_only_new,
            "listing": listing,
            "total_discovered": discovered_total,
            "cars_upserted": cars_upserted,
            "cars_failed": cars_failed,
            "cars_filtered": cars_filtered,
            "images_upserted": images_upserted,
            "protection_events": protection_events,
            "anti_bot_detected": anti_bot_detected,
            "fail_ratio": round(fail_ratio, 4),
            "protection_ratio": round(protection_ratio, 4),
            "skipped_existing": skipped_existing,
            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
            "failures": failures,
        }

    def sync_listing_segmented(
        self,
        segments: list[dict[str, Any]],
        lane: str = "dubizzle_cars",
        only_new: bool | None = None,
        start_segment: int = 0,
        start_page: int = 1,
        progress_callback: Callable[[int], None] | None = None,
    ) -> dict[str, Any]:
        """Итеративный sync_listing по списку сегментов (бренд / бренд+годы).

        Args:
            segments: список dict с ключами make, year_min, year_max.
            start_segment: индекс сегмента для resume (0-based).
            start_page: не используется (остаётся для обратной совместимости API).
            progress_callback: вызывается (segment_index) после каждого ПОЛНОСТЬЮ пройденного сегмента.
        """
        trace_id = self._new_trace_id("sync-segmented")
        started_at = time.perf_counter()
        base_url = self.settings.listing.cars_url
        _ = start_page  # обратная совместимость — page-level resume удалён

        total_cars_upserted = 0
        total_cars_failed = 0
        total_images_upserted = 0
        total_skipped = 0
        total_discovered = 0
        all_failures: list[dict[str, str]] = []
        segment_results: list[dict[str, Any]] = []
        completed_all = True
        skipped_segments = 0
        aggregated_active_urls: set[str] = set()
        aggregated_active_ids: set[str] = set()

        # В segmented-режиме полный прогон должен проходить ВСЕ сегменты.
        # Runtime-фильтры применяются позже (на уровне конкретных карточек),
        # но не должны сужать сам обход сегментов.

        logger.warning(
            "Starting segmented sync: %d segments, resume from segment=%d",
            len(segments), start_segment,
        )

        for seg_idx in range(start_segment, len(segments)):
            seg = segments[seg_idx]
            seg_make = seg.get("make")
            seg_year_min = seg.get("year_min")
            seg_year_max = seg.get("year_max")

            # На длительном full-scan сегмент нельзя пропускать из-за runtime include.brands,
            # иначе прогон становится частичным.
            self._reload_runtime_config()

            seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None

            seg_label = f"{seg_make or 'ALL'}"
            if seg_year_min is not None or seg_year_max is not None:
                seg_label += f" ({seg_year_min}-{seg_year_max})"

            logger.warning(
                "Segment %d/%d: %s",
                seg_idx + 1, len(segments), seg_label,
            )

            outer_progress_callback = self._progress_callback

            def _segment_progress(stage: str, meta: dict[str, Any]) -> None:
                if outer_progress_callback is None:
                    return
                outer_progress_callback(
                    stage,
                    {
                        **meta,
                        "segment_index": seg_idx,
                        "segment_label": seg_label,
                        "segments_total": len(segments),
                    },
                )

            try:
                self.set_progress_callback(_segment_progress)
                self._report_progress(
                    "segment_started",
                    segment_index=seg_idx,
                    segment_label=seg_label,
                    segments_total=len(segments),
                )
                result = self.sync_listing(
                    make=None if seg_url else seg_make,
                    model=None,
                    lane=lane,
                    only_new=only_new,
                    listing_url=seg_url,
                    year_min=seg_year_min,
                    year_max=seg_year_max,
                    skip_mark_sold=True,
                )
                total_cars_upserted += result.get("cars_upserted", 0)
                total_cars_failed += result.get("cars_failed", 0)
                total_images_upserted += result.get("images_upserted", 0)
                total_skipped += result.get("skipped_existing", 0)
                total_discovered += result.get("listing", {}).get("vehicles_collected", 0)
                all_failures.extend(result.get("failures", []))
                aggregated_active_urls.update(result.get("all_listing_origin_urls", set()) or set())
                aggregated_active_ids.update(result.get("all_listing_origin_ids", set()) or set())
                segment_results.append({
                    "segment": seg,
                    "segment_index": seg_idx,
                    "status": result.get("status"),
                    "full_scan_completed": bool(result.get("full_scan_completed", False)),
                    "cars_upserted": result.get("cars_upserted", 0),
                    "cars_failed": result.get("cars_failed", 0),
                    "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0),
                })

                segment_done = bool(result.get("full_scan_completed", False))
                if not segment_done:
                    completed_all = False
                # Даже если сегмент завершился неполно (например, страница/пагинация сломалась),
                # в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше.
                if not segment_done:
                    skipped_segments += 1
                    logger.warning(
                        "Segment %d/%d incomplete: %s — advancing checkpoint and continuing",
                        seg_idx + 1, len(segments), seg_label,
                    )

                # Сегмент обработан до конечного состояния — фиксируем чекпоинт,
                # даже если он был пропущен/оборван с ошибками.
                if progress_callback is not None:
                    try:
                        progress_callback(seg_idx)
                    except Exception:
                        logger.warning(
                            "Failed to persist segment checkpoint for segment=%d",
                            seg_idx, exc_info=True,
                        )

                logger.warning(
                    "Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d",
                    seg_idx + 1, len(segments), seg_label,
                    result.get("cars_upserted", 0),
                    result.get("cars_failed", 0),
                    result.get("listing", {}).get("vehicles_collected", 0),
                )
                self._report_progress(
                    "segment_done",
                    segment_index=seg_idx,
                    segment_label=seg_label,
                    segments_total=len(segments),
                    segment_status=result.get("status"),
                    segment_full_scan_completed=segment_done,
                )
            except Exception as exc:
                completed_all = False
                logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc)
                all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)})
                skipped_segments += 1
                segment_results.append({
                    "segment": seg,
                    "segment_index": seg_idx,
                    "status": "skipped_error",
                    "full_scan_completed": False,
                    "cars_upserted": 0,
                    "cars_failed": 0,
                    "vehicles_collected": 0,
                })
                if progress_callback is not None:
                    try:
                        progress_callback(seg_idx)
                    except Exception:
                        logger.warning(
                            "Failed to persist skipped segment checkpoint for segment=%d",
                            seg_idx, exc_info=True,
                        )
                self._report_progress(
                    "segment_failed",
                    segment_index=seg_idx,
                    segment_label=seg_label,
                    segments_total=len(segments),
                    error=str(exc),
                )
                # Продолжаем оставшиеся сегменты — одна ошибка не должна убивать весь прогон
                continue
            finally:
                self.set_progress_callback(outer_progress_callback)

        if completed_all:
            try:
                if aggregated_active_ids:
                    self.persistence.mark_sold_not_in_listing(aggregated_active_ids)
                elif aggregated_active_urls:
                    self.persistence.mark_sold_not_in_listing_by_urls(aggregated_active_urls)
            except Exception:
                logger.warning("Segmented sync final sold-mark failed", exc_info=True)

        elapsed = round(time.perf_counter() - started_at, 3)
        status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed"

        logger.warning(
            "Segmented sync done: %d/%d segments, upserted=%d, failed=%d, discovered=%d, elapsed=%.1fs",
            len(segment_results), len(segments),
            total_cars_upserted, total_cars_failed, total_discovered, elapsed,
        )

        return {
            "trace_id": trace_id,
            "status": status,
            # Bootstrap считается завершённым если все сегменты пройдены,
            # даже если часть машин failed (они будут обновлены в следующих циклах).
            "full_scan_completed": completed_all,
            "segments_total": len(segments),
            "segments_completed": len(segment_results),
            "segments_skipped": skipped_segments,
            "cars_upserted": total_cars_upserted,
            "cars_failed": total_cars_failed,
            "images_upserted": total_images_upserted,
            "skipped_existing": total_skipped,
            "total_discovered": total_discovered,
            "all_listing_origin_urls": aggregated_active_urls,
            "all_listing_origin_ids": aggregated_active_ids,
            "elapsed_seconds": elapsed,
            "failures": all_failures,
            "segment_results": segment_results,
        }

    def run_scheduled(self) -> None:
        """Standalone-планировщик (в продакшене используется Celery beat)."""
        def _handle_shutdown(signum, frame):
            logger.info("Received signal %s, shutting down gracefully...", signum)
            self._shutdown_requested = True

        signal.signal(signal.SIGINT, _handle_shutdown)
        signal.signal(signal.SIGTERM, _handle_shutdown)

        interval = self.settings.scheduler_interval_minutes * 60
        logger.info(
            "Scheduler started: syncing every %d minutes",
            self.settings.scheduler_interval_minutes,
        )
        cycle = 0
        while not self._shutdown_requested:
            cycle += 1
            logger.info("Scheduler cycle #%d starting", cycle)
            start = time.time()
            try:
                if self.context:
                    try:
                        self.context.close()
                    except PlaywrightError:
                        pass
                    self.context = None

                if self.browser is None or self.playwright is None:
                    logger.info("Browser/Playwright not available, re-initializing...")
                    self.close()
                    self.__enter__()

                result = self.sync_listing()
                elapsed = time.time() - start
                logger.info(
                    "Cycle #%d done in %.1fs: %d upserted, %d failed",
                    cycle, elapsed,
                    result.get("cars_upserted", 0),
                    result.get("cars_failed", 0),
                )
            except Exception as exc:
                elapsed = time.time() - start
                logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc)
                try:
                    self.close()
                except Exception:
                    pass
                try:
                    self.__enter__()
                except Exception as reinit_exc:
                    logger.error("Failed to re-initialize browser: %s", reinit_exc)

            if self._shutdown_requested:
                break

            sleep_time = max(0, interval - (time.time() - start))
            if sleep_time > 0:
                logger.info("Sleeping %.0f seconds until next cycle...", sleep_time)
                slept = 0.0
                while slept < sleep_time and not self._shutdown_requested:
                    chunk = min(5.0, sleep_time - slept)
                    time.sleep(chunk)
                    slept += chunk

        logger.info("Scheduler stopped gracefully after %d cycles.", cycle)