initial openlane project

openlane_scraper/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

openlane_scraper/api/__init__.py

@@ -0,0 +1,3 @@
+from .app import create_app
+
+__all__ = ["create_app"]

openlane_scraper/api/app.py

@@ -0,0 +1,40 @@
+# Создание FastAPI-приложения и настройка его жизненного цикла.
+
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from ..core.config import Settings
+from ..storage.db import PersistenceService
+from .routes import cars, health, tasks
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Жизненный цикл.
+    # Таблицы создаются через Alembic-миграции (сервис migrate).
+    yield
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    _settings = settings or Settings()
+
+    app = FastAPI(
+        title="OpenLane Scraper API",
+        description="REST API для управления задачами скрапинга OpenLane и просмотра данных",
+        version="1.0.0",
+        lifespan=lifespan,
+    )
+
+    app.state.settings = _settings
+    app.state.persistence = PersistenceService(_settings)
+
+    app.include_router(health.router, tags=["health"])
+    app.include_router(cars.router, prefix="/api/v1", tags=["cars"])
+    app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"])
+
+    return app
+
+
+# Экземпляр приложения для запуска через uvicorn.
+app = create_app()

openlane_scraper/api/deps.py

@@ -0,0 +1,9 @@
+# Вспомогательные зависимости для FastAPI-роутов.
+
+from fastapi import Request
+
+from ..storage.db import PersistenceService
+
+
+def get_persistence(request: Request) -> PersistenceService:
+    return request.app.state.persistence

openlane_scraper/api/routes/cars.py

@@ -0,0 +1,103 @@
+# Роуты для просмотра автомобилей и агрегированной статистики.
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import func, select
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import Car, Image
+from ...storage.schemas import CarRead
+
+router = APIRouter()
+
+
+@router.get("/cars")
+def list_cars(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    brand: str | None = None,
+    model: str | None = None,
+    year_min: int | None = None,
+    year_max: int | None = None,
+    is_sold: bool | None = None,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Список автомобилей с пагинацией и фильтрами
+    with persistence.session_scope() as session:
+        query = select(Car)
+
+        if brand:
+            query = query.where(Car.brand.ilike(f"%{brand}%"))
+        if model:
+            query = query.where(Car.model.ilike(f"%{model}%"))
+        if year_min is not None:
+            query = query.where(Car.year >= year_min)
+        if year_max is not None:
+            query = query.where(Car.year <= year_max)
+        if is_sold is not None:
+            query = query.where(Car.is_sold == is_sold)
+
+        count_query = select(func.count()).select_from(query.subquery())
+        total = session.execute(count_query).scalar() or 0
+
+        offset = (page - 1) * per_page
+        cars = session.execute(
+            query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "pages": (total + per_page - 1) // per_page if per_page else 0,
+            "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars],
+        }
+
+
+@router.get("/cars/{car_id}")
+def get_car(
+    car_id: int,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Детальная информация об автомобиле с изображениями
+    with persistence.session_scope() as session:
+        car = session.get(Car, car_id)
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return CarRead.model_validate(car).model_dump(mode="json")
+
+
+@router.get("/cars/by-origin/{origin_id}")
+def get_car_by_origin(
+    origin_id: str,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Поиск автомобиля по origin_id
+    with persistence.session_scope() as session:
+        car = session.execute(
+            select(Car).where(Car.origin_id == origin_id)
+        ).scalars().first()
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return CarRead.model_validate(car).model_dump(mode="json")
+
+
+@router.get("/stats")
+def get_stats(persistence: PersistenceService = Depends(get_persistence)):
+    # Общая статистика по БД
+    with persistence.session_scope() as session:
+        total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
+        total_images = session.execute(select(func.count(Image.id))).scalar() or 0
+        brands = session.execute(
+            select(Car.brand, func.count(Car.id))
+            .group_by(Car.brand)
+            .order_by(func.count(Car.id).desc())
+            .limit(20)
+        ).all()
+
+        return {
+            "total_cars": total_cars,
+            "total_images": total_images,
+            "top_brands": [{"brand": b, "count": c} for b, c in brands],
+        }
+

openlane_scraper/api/routes/health.py

@@ -0,0 +1,30 @@
+# Роут проверки доступности сервиса и соединения с БД.
+
+import logging
+
+from fastapi import APIRouter, Depends
+from sqlalchemy import text
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+
+router = APIRouter()
+logger = logging.getLogger("openlane_scraper.api.health")
+
+
+@router.get("/health")
+def health_check(persistence: PersistenceService = Depends(get_persistence)):
+    # Проверка API и БД
+    db_ok = False
+    try:
+        with persistence.session_scope() as session:
+            session.execute(text("SELECT 1"))
+            db_ok = True
+    except Exception:
+        logger.warning("Health DB check failed", exc_info=True)
+
+    return {
+        "status": "ok" if db_ok else "degraded",
+        "service": "openlane-scraper-api",
+        "database": "connected" if db_ok else "unavailable",
+    }

openlane_scraper/api/routes/tasks.py

@@ -0,0 +1,96 @@
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from sqlalchemy import select, func
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import SyncRun
+from ...worker.celery_app import celery_app
+from ...worker.tasks import sync_listing_task
+
+router = APIRouter()
+
+
+class SyncListingRequest(BaseModel):
+    lane: str = "openlane_marketplace"
+    limit: int | None = None
+    only_new: bool | None = None
+    max_pages: int | None = None
+    concurrency: int | None = None
+
+
+@router.post("/tasks/sync-listing")
+def start_sync_listing(
+    body: SyncListingRequest,
+):
+    result = sync_listing_task.apply_async(
+        kwargs={
+            "lane": body.lane,
+            "limit": body.limit,
+            "only_new": body.only_new,
+            "max_pages": body.max_pages,
+            "concurrency": body.concurrency,
+        },
+        queue="scraping",
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+    }
+
+
+@router.get("/tasks/{task_id}")
+def get_task_status(task_id: str):
+    result = celery_app.AsyncResult(task_id)
+
+    payload: dict = {
+        "task_id": task_id,
+        "state": result.state,
+    }
+
+    if result.successful():
+        payload["result"] = result.result
+    elif result.failed():
+        payload["error"] = str(result.result)
+    elif result.info is not None:
+        payload["meta"] = result.info
+
+    return payload
+
+
+@router.get("/sync-runs")
+def list_sync_runs(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    with persistence.session_scope() as session:
+        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+        offset = (page - 1) * per_page
+        runs = session.execute(
+            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": r.id,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
+                    "status": r.status,
+                    "lane": r.lane,
+                    "ids_fetched": r.ids_fetched,
+                    "cars_upserted": r.cars_upserted,
+                    "cars_failed": r.cars_failed,
+                    "images_upserted": r.images_upserted,
+                    "error_summary": r.error_summary,
+                }
+                for r in runs
+            ],
+        }

openlane_scraper/browser/__init__.py

@@ -0,0 +1,3 @@
+from .factory import BrowserFactory
+
+__all__ = ["BrowserFactory"]

openlane_scraper/browser/factory.py

@@ -0,0 +1,226 @@
+import json
+import logging
+import random
+
+from playwright.sync_api import Browser, BrowserContext, Playwright
+
+try:
+    from playwright_stealth import stealth_sync
+except ImportError:
+    stealth_sync = None
+
+from ..core.config import Settings
+
+logger = logging.getLogger("openlane_scraper.browser")
+
+
+def _build_init_script() -> str:
+    # Патч признаков автоматизации.
+    hardware_concurrency = random.choice([4, 8, 12, 16])
+    device_memory = random.choice([4, 8, 16])
+    languages = ["en-US", "en"]
+
+    return f"""
+(() => {{
+  const define = (obj, prop, value) => {{
+    try {{
+      Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
+    }} catch (e) {{}}
+  }};
+
+  define(navigator, 'webdriver', undefined);
+  define(navigator, 'platform', 'Win32');
+  define(navigator, 'vendor', 'Google Inc.');
+  define(navigator, 'language', '{languages[0]}');
+  define(navigator, 'languages', {json.dumps(languages)});
+  define(navigator, 'hardwareConcurrency', {hardware_concurrency});
+  define(navigator, 'deviceMemory', {device_memory});
+  define(navigator, 'maxTouchPoints', 0);
+
+  if (!window.chrome) {{
+    Object.defineProperty(window, 'chrome', {{
+      value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
+      configurable: true
+    }});
+  }}
+
+  const originalQuery = navigator.permissions && navigator.permissions.query;
+  if (originalQuery) {{
+    navigator.permissions.query = (params) => (
+      params && params.name === 'notifications'
+        ? Promise.resolve({{ state: Notification.permission }})
+        : originalQuery(params)
+    );
+  }}
+
+  const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
+  WebGLRenderingContext.prototype.getParameter = function(parameter) {{
+    if (parameter === 37445) return 'Intel Inc.';
+    if (parameter === 37446) return 'Intel Iris OpenGL Engine';
+    return originalGetParameter.call(this, parameter);
+  }};
+}})();
+"""
+
+
+class BrowserFactory:
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def _resolve_engine(self) -> str:
+        # Выбор движка браузера.
+        engine = self.settings.browser_engine.strip().lower()
+        if engine == "auto":
+            # В headless-режиме Chromium со stealth-скриптами
+            # значительно стабильнее Firefox по anti-bot.
+            return "chromium"
+        if engine in ("firefox", "chromium"):
+            return engine
+        logger.warning("Unknown OPENLANE_BROWSER_ENGINE=%r, falling back to auto", engine)
+        return "chromium"
+
+    def create_browser(self, playwright: Playwright) -> Browser:
+        engine = self._resolve_engine()
+        proxy_dict = self.settings.proxy.to_playwright_dict()
+        logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
+
+        if engine == "firefox":
+            launch_kwargs: dict = {"headless": self.settings.headless}
+            if proxy_dict:
+                launch_kwargs["proxy"] = proxy_dict
+                logger.info("Using proxy: %s", self.settings.proxy.server)
+            # Параметры Firefox для headless-режима.
+            launch_kwargs["firefox_user_prefs"] = {
+                "dom.webdriver.enabled": False,
+                "useAutomationExtension": False,
+                # Базовые оптимизации для VPS.
+                "media.autoplay.default": 5,
+                "media.volume_scale": "0.0",
+                "media.audio.playback.standalone": False,
+                "dom.ipc.processCount": 1,
+                "dom.ipc.plugins.enabled": False,
+                "browser.cache.disk.enable": False,
+                "browser.cache.memory.enable": True,
+                "browser.cache.memory.max_entry_size": 8192,
+                "network.prefetch-next": False,
+                "network.dns.disablePrefetch": True,
+                "permissions.default.image": 2,
+                "javascript.options.mem.gc_incremental_mark_slice_ms": 20,
+            }
+            logger.info("Launching Firefox (headless=%s)", self.settings.headless)
+            return playwright.firefox.launch(**launch_kwargs)
+
+        # Путь запуска Chromium.
+        args = [
+            "--disable-blink-features=AutomationControlled",
+            "--no-default-browser-check",
+            "--disable-dev-shm-usage",
+            "--disable-features=IsolateOrigins,site-per-process",
+        ]
+        if self.settings.headless:
+            args.append("--headless=new")
+            pw_headless = False
+            logger.info("Using Chromium new-headless mode (--headless=new)")
+        else:
+            pw_headless = False
+
+        launch_kwargs = {"headless": pw_headless, "args": args}
+        if proxy_dict:
+            launch_kwargs["proxy"] = proxy_dict
+            logger.info("Using proxy: %s", self.settings.proxy.server)
+        try:
+            logger.info("Trying to launch real Chrome channel")
+            return playwright.chromium.launch(channel="chrome", **launch_kwargs)
+        except Exception:
+            logger.warning("Chrome channel launch failed, falling back to Chromium")
+            return playwright.chromium.launch(**launch_kwargs)
+
+    def create_context(self, browser: Browser, storage_state_path: str | None = None) -> BrowserContext:
+        viewport = random.choice(self.settings.fingerprint.viewport_presets)
+        timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
+        color_scheme = random.choice(["light", "dark"])
+
+        is_firefox = browser.browser_type.name == "firefox"
+
+        ctx_kwargs: dict = {
+            "viewport": viewport,
+            "screen": viewport,
+            "locale": self.settings.fingerprint.locale,
+            "timezone_id": timezone_id,
+            "color_scheme": color_scheme,
+            "java_script_enabled": True,
+            "ignore_https_errors": False,
+        }
+
+        if is_firefox:
+            # Заголовки и user-agent для Firefox.
+            ctx_kwargs["user_agent"] = (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
+                "Gecko/20100101 Firefox/128.0"
+            )
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.5",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+            }
+        else:
+            ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
+            ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
+            ctx_kwargs["is_mobile"] = False
+            ctx_kwargs["has_touch"] = False
+            ctx_kwargs["extra_http_headers"] = {
+                "Accept-Language": "en-US,en;q=0.9",
+                "DNT": "1",
+                "Upgrade-Insecure-Requests": "1",
+                "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
+                "Sec-CH-UA-Mobile": "?0",
+                "Sec-CH-UA-Platform": '"Windows"',
+            }
+
+        if storage_state_path:
+            ctx_kwargs["storage_state"] = storage_state_path
+
+        context = browser.new_context(**ctx_kwargs)
+        context.set_default_timeout(self.settings.default_timeout_ms)
+        context.set_default_navigation_timeout(self.settings.default_timeout_ms)
+
+        if not is_firefox:
+            # Патчи маскировки для Chromium.
+            context.add_init_script(_build_init_script())
+            if stealth_sync:
+                context.on("page", lambda page: stealth_sync(page))
+                logger.debug("playwright-stealth attached to context")
+
+        return context
+
+    @staticmethod
+    def enable_resource_blocking(page) -> None:
+        # Блокируем всё кроме document и XHR/fetch — минимальный след.
+        BLOCKED_TYPES = {"image", "stylesheet", "font", "media", "websocket", "eventsource", "manifest", "other"}
+        BLOCKED_URL_PATTERNS = (
+            "google-analytics", "googletagmanager", "facebook.net",
+            "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
+            "analytics", "tracking", "adservice",
+            # OpenLane-трекеры из сетевого лога.
+            "bugsnag", "sessions.bugsnag",
+            "intercom", "widget.intercom",
+            "pusher", "sockjs",
+            "segment", "cdn.segment",
+            "ap3.com", "ap3c.com",
+            "onetrust", "optanon", "cookielaw",
+            "fullstory", "sentry",
+        )
+
+        def _handle_route(route):
+            req = route.request
+            if req.resource_type in BLOCKED_TYPES:
+                route.abort()
+                return
+            url = req.url.lower()
+            if any(pat in url for pat in BLOCKED_URL_PATTERNS):
+                route.abort()
+                return
+            route.continue_()
+
+        page.route("**/*", _handle_route)

openlane_scraper/cli.py

@@ -0,0 +1,92 @@
+import argparse
+from pathlib import Path
+
+from .core.config import Settings
+from .core.utils import save_to_json
+from .openlane import OpenLaneScrapeRunner
+from .scraper import OpenLaneScraper
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="OpenLane scraper CLI")
+    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
+    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    subparsers.add_parser("init-db", help="Create DB tables")
+
+    sync_parser = subparsers.add_parser("sync-listing", help="Sync OpenLane marketplace → DB")
+    sync_parser.add_argument("--lane", default="openlane_marketplace")
+    sync_parser.add_argument("--limit", type=int, default=None)
+    sync_parser.add_argument("--only-new", choices=["true", "false"], default=None)
+    sync_parser.add_argument("--max-pages", type=int, default=None)
+    sync_parser.add_argument("--concurrency", type=int, default=None)
+    sync_parser.add_argument("--output", default=str(Path("artifacts/json/openlane_sync_listing.json")))
+
+    scrape_parser = subparsers.add_parser("scrape-openlane", help="Scrape OpenLane marketplace to JSONL files")
+    scrape_parser.add_argument("--max-pages", type=int, default=None)
+    scrape_parser.add_argument("--concurrency", type=int, default=None)
+    scrape_parser.add_argument("--resume", action="store_true")
+    scrape_parser.add_argument("--checkpoint-every-pages", type=int, default=None)
+
+    subparsers.add_parser("openlane-login", help="Open interactive OpenLane login and persist browser storage state")
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    runtime_settings: Settings | None = None
+    if args.headless is not None or args.debug:
+        runtime_settings = Settings()
+        if args.headless is not None:
+            runtime_settings.headless = args.headless == "true"
+        if args.debug:
+            runtime_settings.log_level = "DEBUG"
+
+    if args.command == "openlane-login":
+        runner = OpenLaneScrapeRunner(runtime_settings)
+        storage_state_path = runner.interactive_login()
+        print(f"OpenLane storage state saved to {Path(storage_state_path).resolve()}")
+        return
+
+    if args.command == "scrape-openlane":
+        runner = OpenLaneScrapeRunner(runtime_settings)
+        result = runner.run(
+            max_pages=args.max_pages,
+            concurrency=args.concurrency,
+            resume=args.resume,
+            checkpoint_every_pages=args.checkpoint_every_pages,
+        )
+        print(f"OpenLane JSONL saved to {Path(result.jsonl_path).resolve()}")
+        print(f"OpenLane aggregated JSON saved to {Path(result.aggregated_path).resolve()}")
+        print(f"OpenLane checkpoint saved to {Path(result.checkpoint_path).resolve()}")
+        print(f"OpenLane storage state saved to {Path(result.storage_state_path).resolve()}")
+        print(
+            f"OpenLane completed pages={len(result.completed_pages)} failed pages={len(result.failed_pages)} total_records={result.total_records} elapsed={result.elapsed_seconds:.2f}s"
+        )
+        return
+
+    with OpenLaneScraper(runtime_settings) as scraper:
+        if args.command == "init-db":
+            data = scraper.init_db()
+            print(f"DB initialized: {data}")
+            return
+        elif args.command == "sync-listing":
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+                max_pages=args.max_pages,
+                concurrency=args.concurrency,
+            )
+
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+
+
+if __name__ == "__main__":
+    main()

openlane_scraper/core/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

openlane_scraper/core/config.py

@@ -0,0 +1,194 @@
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+# --- Конфиг OpenLane (auth, paging, throttle, retry, checkpoint/output) ---
+
+@dataclass(slots=True)
+class OpenLaneConfig:
+    sign_in_url: str = _env_str("OPENLANE_SIGN_IN_URL", "https://app.openlane.com/sign_in")
+    search_url: str = _env_str("OPENLANE_SEARCH_URL", "https://app.openlane.com/search?tab=marketplace")
+    api_search_url: str = _env_str("OPENLANE_API_SEARCH_URL", "https://app.openlane.com/api/v4/search")
+    username: str = _env_str("OPENLANE_USERNAME", "")
+    password: str = _env_str("OPENLANE_PASSWORD", "")
+    source_tab: str = _env_str("OPENLANE_SOURCE_TAB", "marketplace")
+    sale_types: str = _env_str("OPENLANE_SALE_TYPES", "marketplace")
+    page_size: int = _env_int("OPENLANE_PAGE_SIZE", 30)
+    max_pages: int = _env_int("OPENLANE_MAX_PAGES", 512)
+    concurrency: int = _env_int("OPENLANE_CONCURRENCY", 4)
+    throttle_min_seconds: float = _env_float("OPENLANE_THROTTLE_MIN_SECONDS", 0.3)
+    throttle_max_seconds: float = _env_float("OPENLANE_THROTTLE_MAX_SECONDS", 0.8)
+    retry_schedule_seconds: tuple[float, ...] = tuple(
+        float(part.strip())
+        for part in _env_str("OPENLANE_RETRY_SCHEDULE_SECONDS", "2,5,10").split(",")
+        if part.strip()
+    ) or (2.0, 5.0, 10.0)
+    checkpoint_every_pages: int = _env_int("OPENLANE_CHECKPOINT_EVERY_PAGES", 10)
+    output_dir: str = _env_str("OPENLANE_OUTPUT_DIR", "artifacts/openlane")
+    jsonl_output: str = _env_str("OPENLANE_JSONL_OUTPUT", "artifacts/openlane/openlane_search.jsonl")
+    aggregated_output: str = _env_str("OPENLANE_AGGREGATED_OUTPUT", "artifacts/openlane/openlane_search_aggregated.json")
+    checkpoint_file: str = _env_str("OPENLANE_CHECKPOINT_FILE", "artifacts/openlane/openlane_checkpoint.json")
+    storage_state_file: str = _env_str("OPENLANE_STORAGE_STATE_FILE", "artifacts/openlane/openlane_storage_state.json")
+    persist_storage_state: bool = _env_bool("OPENLANE_PERSIST_STORAGE_STATE", True)
+    progress_log_every_pages: int = _env_int("OPENLANE_PROGRESS_LOG_EVERY_PAGES", 10)
+    request_timeout_ms: int = _env_int("OPENLANE_REQUEST_TIMEOUT_MS", 45000)
+    max_cars_limit: int = _env_int("OPENLANE_MAX_CARS_LIMIT", 20)
+    refresh_token: str = _env_str("OPENLANE_REFRESH_TOKEN", "")
+    okta_client_id: str = _env_str("OPENLANE_OKTA_CLIENT_ID", "")
+    okta_token_endpoint: str = _env_str(
+        "OPENLANE_OKTA_TOKEN_ENDPOINT",
+        "https://okta.iam.karglobal.com/oauth2/default/v1/token",
+    )
+    okta_redirect_uri: str = _env_str("OPENLANE_OKTA_REDIRECT_URI", "https://app.openlane.com/sign_in")
+    okta_timeout_seconds: int = _env_int("OPENLANE_OKTA_TIMEOUT_SECONDS", 15)
+
+
+# --- Конфиг PostgreSQL (URL, пул соединений, pool_recycle) ---
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("OPENLANE_DATABASE_URL", "postgresql+psycopg2://openlane:openlane@localhost:5432/openlane_scraper")
+    echo: bool = _env_bool("OPENLANE_DATABASE_ECHO", False)
+    pool_size: int = _env_int("OPENLANE_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("OPENLANE_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("OPENLANE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("OPENLANE_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("OPENLANE_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("OPENLANE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+
+
+# --- Конфиг прокси (server, username, password) ---
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("OPENLANE_PROXY_SERVER")
+    username: str | None = _env_optional_str("OPENLANE_PROXY_USERNAME")
+    password: str | None = _env_optional_str("OPENLANE_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
+#  Главный объект настроек: собирает все блоки конфигурации 
+
+@dataclass(slots=True)
+class Settings:
+    default_timeout_ms: int = _env_int("OPENLANE_TIMEOUT_MS", 45000)
+    headless: bool = _env_bool("OPENLANE_HEADLESS", True)
+    browser_engine: str = _env_str("OPENLANE_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("OPENLANE_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("OPENLANE_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("OPENLANE_ENABLE_TRACE_ID_LOGS", True)
+    runtime_config_file: str | None = _env_path_str("OPENLANE_RUNTIME_CONFIG_FILE")
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    openlane: OpenLaneConfig = field(default_factory=OpenLaneConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()

openlane_scraper/core/logs.py

@@ -0,0 +1,39 @@
+import logging
+import sys
+from contextvars import ContextVar
+
+# Храним trace_id текущего потока/корутины.
+TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
+
+
+class TraceIdFilter(logging.Filter):
+    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.trace_id = TRACE_ID.get()
+        return True
+
+
+def set_trace_id(trace_id: str) -> None:
+    TRACE_ID.set(trace_id)
+
+
+def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
+    # stderr — Docker и Celery prefork корректно его подхватывают.
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
+    trace_filter = TraceIdFilter()
+    for handler in handlers:
+        handler.addFilter(trace_filter)
+        handler.setLevel(getattr(logging, level.upper(), logging.INFO))
+    root = logging.getLogger()
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))
+    # Убираем старые хендлеры, чтобы не дублировать после fork.
+    root.handlers.clear()
+    for handler in handlers:
+        root.addHandler(handler)
+    fmt = logging.Formatter(
+        "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
+    )
+    for handler in root.handlers:
+        handler.setFormatter(fmt)

openlane_scraper/core/runtime_config.py

@@ -0,0 +1,379 @@
+"""Runtime config: загрузка и применение runtime_config.json.
+
+Позволяет менять параметры sync и фильтры машин без перезапуска сервиса.
+Файл перечитывается при каждом запуске sync_listing.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("openlane_scraper.core.runtime_config")
+
+
+@dataclass(slots=True)
+class SyncConfig:
+    name: str | None = None
+    ids_initial_size: int | None = None
+    ids_next_size: int | None = None
+    ids_max_pages: int | None = None
+    condition_check_enabled: bool | None = None
+    lane: str = "openlane_marketplace"
+    only_new: bool = False
+    limit: int | None = None
+
+
+@dataclass(slots=True)
+class ListingConfig:
+    make: str | None = None
+    model: str | None = None
+
+
+@dataclass(slots=True)
+class RangeFilter:
+    min: int | None = None
+    max: int | None = None
+
+
+@dataclass(slots=True)
+class FlagFilters:
+    damaged_only: bool | None = None
+    run_and_drive: bool | None = None
+
+
+@dataclass(slots=True)
+class FieldFilters:
+    brands: list[str] = field(default_factory=list)
+    models: list[str] = field(default_factory=list)
+    years: list[int] = field(default_factory=list)
+    body_types: list[str] = field(default_factory=list)
+    colors: list[str] = field(default_factory=list)
+    drives: list[str] = field(default_factory=list)
+    gearboxes: list[str] = field(default_factory=list)
+    locations: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class Filters:
+    price: RangeFilter = field(default_factory=RangeFilter)
+    mileage: RangeFilter = field(default_factory=RangeFilter)
+    flags: FlagFilters = field(default_factory=FlagFilters)
+
+    include: FieldFilters = field(default_factory=FieldFilters)
+    exclude: FieldFilters = field(default_factory=FieldFilters)
+
+    # Legacy flat fields (backward compatibility).
+    brands: list[str] = field(default_factory=list)
+    models: list[str] = field(default_factory=list)
+    years: list[int] = field(default_factory=list)
+    body_types: list[str] = field(default_factory=list)
+    colors: list[str] = field(default_factory=list)
+    drives: list[str] = field(default_factory=list)
+    gearboxes: list[str] = field(default_factory=list)
+    locations: list[str] = field(default_factory=list)
+
+    exclude_brands: list[str] = field(default_factory=list)
+    exclude_models: list[str] = field(default_factory=list)
+    exclude_years: list[int] = field(default_factory=list)
+    exclude_body_types: list[str] = field(default_factory=list)
+    exclude_colors: list[str] = field(default_factory=list)
+    exclude_drives: list[str] = field(default_factory=list)
+    exclude_gearboxes: list[str] = field(default_factory=list)
+    exclude_locations: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class RuntimeConfig:
+    sync: SyncConfig = field(default_factory=SyncConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    filters: Filters = field(default_factory=Filters)
+
+
+def load_runtime_config(path: str | Path | None) -> RuntimeConfig:
+    """Загрузить runtime_config.json. Если файл отсутствует — вернуть дефолты."""
+    if not path:
+        return RuntimeConfig()
+
+    p = Path(path)
+    if not p.exists():
+        logger.debug("runtime_config not found at %s — using defaults", p)
+        return RuntimeConfig()
+
+    try:
+        raw = json.loads(p.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError) as exc:
+        logger.warning("Failed to parse runtime_config %s: %s — using defaults", p, exc)
+        return RuntimeConfig()
+
+    cfg = RuntimeConfig()
+
+    sync_raw = raw.get("sync") or {}
+    if isinstance(sync_raw, dict):
+        cfg.sync.name = _opt_str(sync_raw.get("name"))
+        cfg.sync.ids_initial_size = _opt_int(sync_raw.get("ids_initial_size"))
+        cfg.sync.ids_next_size = _opt_int(sync_raw.get("ids_next_size"))
+        cfg.sync.ids_max_pages = _opt_int(sync_raw.get("ids_max_pages"))
+        cfg.sync.condition_check_enabled = _opt_bool(sync_raw.get("condition_check_enabled"))
+        cfg.sync.lane = _opt_str(sync_raw.get("lane")) or cfg.sync.lane
+        only_new = _opt_bool(sync_raw.get("only_new"))
+        if only_new is not None:
+            cfg.sync.only_new = only_new
+        cfg.sync.limit = _opt_int(sync_raw.get("limit"))
+
+    listing_raw = raw.get("listing") or {}
+    if isinstance(listing_raw, dict):
+        cfg.listing.make = _opt_str(listing_raw.get("make"))
+        cfg.listing.model = _opt_str(listing_raw.get("model"))
+
+    filters_raw = raw.get("filters") or {}
+    if isinstance(filters_raw, dict):
+        cfg.filters.price = _parse_range(filters_raw.get("price"))
+        cfg.filters.mileage = _parse_range(filters_raw.get("mileage"))
+        cfg.filters.flags = _parse_flags(filters_raw.get("flags"))
+
+        include_payload = filters_raw.get("include") if isinstance(filters_raw.get("include"), dict) else filters_raw
+        exclude_payload = (
+            filters_raw.get("exclude")
+            if isinstance(filters_raw.get("exclude"), dict)
+            else {
+                "brands": filters_raw.get("exclude_brands"),
+                "models": filters_raw.get("exclude_models"),
+                "years": filters_raw.get("exclude_years"),
+                "body_types": filters_raw.get("exclude_body_types"),
+                "colors": filters_raw.get("exclude_colors"),
+                "drives": filters_raw.get("exclude_drives"),
+                "gearboxes": filters_raw.get("exclude_gearboxes"),
+                "locations": filters_raw.get("exclude_locations"),
+            }
+        )
+
+        cfg.filters.include = _parse_fields(include_payload)
+        cfg.filters.exclude = _parse_fields(exclude_payload)
+
+        # Backward-compatible flat aliases.
+        cfg.filters.brands = list(cfg.filters.include.brands)
+        cfg.filters.models = list(cfg.filters.include.models)
+        cfg.filters.years = list(cfg.filters.include.years)
+        cfg.filters.body_types = list(cfg.filters.include.body_types)
+        cfg.filters.colors = list(cfg.filters.include.colors)
+        cfg.filters.drives = list(cfg.filters.include.drives)
+        cfg.filters.gearboxes = list(cfg.filters.include.gearboxes)
+        cfg.filters.locations = list(cfg.filters.include.locations)
+
+        cfg.filters.exclude_brands = list(cfg.filters.exclude.brands)
+        cfg.filters.exclude_models = list(cfg.filters.exclude.models)
+        cfg.filters.exclude_years = list(cfg.filters.exclude.years)
+        cfg.filters.exclude_body_types = list(cfg.filters.exclude.body_types)
+        cfg.filters.exclude_colors = list(cfg.filters.exclude.colors)
+        cfg.filters.exclude_drives = list(cfg.filters.exclude.drives)
+        cfg.filters.exclude_gearboxes = list(cfg.filters.exclude.gearboxes)
+        cfg.filters.exclude_locations = list(cfg.filters.exclude.locations)
+
+    logger.info(
+        "Loaded runtime_config from %s: lane=%s, limit=%s",
+        p,
+        cfg.sync.lane,
+        cfg.sync.limit,
+    )
+    return cfg
+
+
+def apply_filters(records: list[dict], filters: Filters) -> list[dict]:
+    """Применить runtime-фильтры к списку сырых записей из API."""
+    if not records:
+        return records
+
+    include = getattr(filters, "include", None)
+    exclude = getattr(filters, "exclude", None)
+
+    include_brands = tuple(getattr(filters, "brands", ()) or getattr(include, "brands", ()))
+    include_models = tuple(getattr(filters, "models", ()) or getattr(include, "models", ()))
+    include_years = tuple(getattr(filters, "years", ()) or getattr(include, "years", ()))
+    include_body_types = tuple(getattr(filters, "body_types", ()) or getattr(include, "body_types", ()))
+
+    exclude_brands = tuple(getattr(filters, "exclude_brands", ()) or getattr(exclude, "brands", ()))
+    exclude_models = tuple(getattr(filters, "exclude_models", ()) or getattr(exclude, "models", ()))
+    exclude_years = tuple(getattr(filters, "exclude_years", ()) or getattr(exclude, "years", ()))
+    exclude_body_types = tuple(getattr(filters, "exclude_body_types", ()) or getattr(exclude, "body_types", ()))
+
+    price_cfg = getattr(filters, "price", None)
+    mileage_cfg = getattr(filters, "mileage", None)
+    flags_cfg = getattr(filters, "flags", None)
+
+    price_min = getattr(price_cfg, "min", None)
+    price_max = getattr(price_cfg, "max", None)
+    mileage_min = getattr(mileage_cfg, "min", None)
+    mileage_max = getattr(mileage_cfg, "max", None)
+    damaged_only = getattr(flags_cfg, "damaged_only", None)
+    run_and_drive = getattr(flags_cfg, "run_and_drive", None)
+
+    result = records
+
+    if include_brands:
+        include_set = {s.casefold() for s in include_brands}
+        result = [r for r in result if _get_brand(r).casefold() in include_set]
+    if exclude_brands:
+        exclude_set = {s.casefold() for s in exclude_brands}
+        result = [r for r in result if _get_brand(r).casefold() not in exclude_set]
+
+    if include_models:
+        include_set = {s.casefold() for s in include_models}
+        result = [r for r in result if _get_model(r).casefold() in include_set]
+    if exclude_models:
+        exclude_set = {s.casefold() for s in exclude_models}
+        result = [r for r in result if _get_model(r).casefold() not in exclude_set]
+
+    if include_years:
+        years_set = set(include_years)
+        result = [r for r in result if _get_year(r) in years_set]
+    if exclude_years:
+        years_set = set(exclude_years)
+        result = [r for r in result if _get_year(r) not in years_set]
+
+    if include_body_types:
+        include_set = {s.casefold() for s in include_body_types}
+        result = [r for r in result if _get_body_type(r).casefold() in include_set]
+    if exclude_body_types:
+        exclude_set = {s.casefold() for s in exclude_body_types}
+        result = [r for r in result if _get_body_type(r).casefold() not in exclude_set]
+
+    if price_min is not None or price_max is not None:
+        filtered: list[dict] = []
+        for r in result:
+            price = _get_price(r)
+            if price is None:
+                filtered.append(r)
+                continue
+            if price_min is not None and price < price_min:
+                continue
+            if price_max is not None and price > price_max:
+                continue
+            filtered.append(r)
+        result = filtered
+
+    if mileage_min is not None or mileage_max is not None:
+        filtered = []
+        for r in result:
+            miles = _get_mileage(r)
+            if miles is None:
+                filtered.append(r)
+                continue
+            if mileage_min is not None and miles < mileage_min:
+                continue
+            if mileage_max is not None and miles > mileage_max:
+                continue
+            filtered.append(r)
+        result = filtered
+
+    if damaged_only is not None:
+        result = [r for r in result if bool(r.get("is_damaged")) is damaged_only]
+    if run_and_drive is not None:
+        result = [r for r in result if bool(r.get("run_and_drive")) is run_and_drive]
+
+    return result
+
+
+def _parse_range(payload: Any) -> RangeFilter:
+    payload = payload if isinstance(payload, dict) else {}
+    return RangeFilter(min=_opt_int(payload.get("min")), max=_opt_int(payload.get("max")))
+
+
+def _parse_flags(payload: Any) -> FlagFilters:
+    payload = payload if isinstance(payload, dict) else {}
+    return FlagFilters(
+        damaged_only=_opt_bool(payload.get("damaged_only")),
+        run_and_drive=_opt_bool(payload.get("run_and_drive")),
+    )
+
+
+def _parse_fields(payload: Any) -> FieldFilters:
+    payload = payload if isinstance(payload, dict) else {}
+    return FieldFilters(
+        brands=_str_list(payload.get("brands")),
+        models=_str_list(payload.get("models")),
+        years=_int_list(payload.get("years")),
+        body_types=_str_list(payload.get("body_types")),
+        colors=_str_list(payload.get("colors")),
+        drives=_str_list(payload.get("drives")),
+        gearboxes=_str_list(payload.get("gearboxes")),
+        locations=_str_list(payload.get("locations")),
+    )
+
+
+def _opt_int(value: Any) -> int | None:
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def _opt_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().casefold()
+        if normalized in {"1", "true", "yes", "on"}:
+            return True
+        if normalized in {"0", "false", "no", "off"}:
+            return False
+    return None
+
+
+def _opt_str(value: Any) -> str | None:
+    if value is None:
+        return None
+    s = str(value).strip()
+    return s or None
+
+
+def _str_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [str(v).strip() for v in value if str(v).strip()]
+
+
+def _int_list(value: Any) -> list[int]:
+    if not isinstance(value, list):
+        return []
+    result: list[int] = []
+    for v in value:
+        iv = _opt_int(v)
+        if iv is not None:
+            result.append(iv)
+    return result
+
+
+def _get_brand(record: dict[str, Any]) -> str:
+    return str(record.get("make") or record.get("brand") or "")
+
+
+def _get_model(record: dict[str, Any]) -> str:
+    return str(record.get("model") or "")
+
+
+def _get_body_type(record: dict[str, Any]) -> str:
+    return str(record.get("body_type") or record.get("body_style") or record.get("vehicle_type") or "")
+
+
+def _get_year(record: dict[str, Any]) -> int | None:
+    return _opt_int(record.get("year"))
+
+
+def _get_price(record: dict[str, Any]) -> int | None:
+    for key in ("current_high_bid", "buy_now_price", "price", "current_bid", "sale_price"):
+        value = _opt_int(record.get(key))
+        if value is not None:
+            return value
+    return None
+
+
+def _get_mileage(record: dict[str, Any]) -> int | None:
+    return _opt_int(record.get("odometer") or record.get("mileage"))

openlane_scraper/core/utils.py

@@ -0,0 +1,66 @@
+import json
+import re
+from pathlib import Path
+from typing import Any, Callable, Iterable
+
+
+def save_to_json(data: Any, filename: str | Path) -> None:
+    path = Path(filename)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def first_non_empty(values: Iterable[Any]) -> Any | None:
+    for value in values:
+        if value not in (None, "", [], {}, ()):
+            return value
+    return None
+
+
+def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
+    found = []
+    if _depth >= max_depth:
+        return found
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if key.lower() in target_keys:
+                found.append(value)
+            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    elif isinstance(obj, list):
+        for item in obj:
+            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    return found
+
+
+def deep_find_all_keys(
+    payloads: list,
+    field_map: dict[str, set[str]],
+    max_depth: int = 64,
+) -> dict[str, list]:
+    """Извлекает все нужные поля за один проход по JSON."""
+    # Готовим обратную карту: нормализованный ключ -> имя поля.
+    reverse: dict[str, str] = {}
+    for field_name, keys in field_map.items():
+        for k in keys:
+            reverse[k.lower()] = field_name
+
+    result: dict[str, list] = {f: [] for f in field_map}
+
+    def _recurse(obj: Any, depth: int) -> None:
+        if depth >= max_depth:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                field = reverse.get(k.lower())
+                if field is not None:
+                    result[field].append(v)
+                _recurse(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _recurse(item, depth + 1)
+
+    for payload in payloads:
+        _recurse(payload, 0)
+
+    return result

openlane_scraper/openlane/__init__.py

@@ -0,0 +1,4 @@
+from .mapper import map_openlane_record, map_openlane_records
+from .runner import OpenLaneScrapeRunner, OpenLaneScrapeResult
+
+__all__ = ["OpenLaneScrapeRunner", "OpenLaneScrapeResult", "map_openlane_record", "map_openlane_records"]

openlane_scraper/openlane/auth.py

@@ -0,0 +1,484 @@
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import os
+import re
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+from playwright.sync_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError
+
+from ..browser.factory import BrowserFactory
+from ..core.config import OpenLaneConfig
+
+logger = logging.getLogger("openlane_scraper.openlane.auth")
+
+# Порог предупреждения об истечении refresh_token (дни).
+_TOKEN_REFRESH_THRESHOLD_DAYS = 3
+_ACCESS_TOKEN_MIN_TTL_SECONDS = 300
+
+
+class OpenLaneAuthError(RuntimeError):
+    pass
+
+
+class OpenLaneAuthenticator:
+    def __init__(self, config: OpenLaneConfig) -> None:
+        self.config = config
+
+    def validate_credentials(self) -> None:
+        if not self.config.username or not self.config.password:
+            raise OpenLaneAuthError(
+                "OPENLANE_USERNAME and OPENLANE_PASSWORD must be set in the environment"
+            )
+
+    def interactive_login_and_persist(self, context: BrowserContext) -> str:
+        page = context.new_page()
+        page.set_default_timeout(self.config.request_timeout_ms)
+        logger.info("OpenLane interactive login: open %s and complete sign-in manually", self.config.sign_in_url)
+        page.goto(self.config.sign_in_url, wait_until="domcontentloaded")
+        self._wait_for_authenticated_session(page)
+        self._persist_storage_state(context)
+        logger.info("OpenLane interactive login successful")
+        return str(Path(self.config.storage_state_file))
+
+    def _inject_refresh_token(self, context: BrowserContext) -> None:
+        # Инъекция refresh_token cookie из env.
+        token = self.config.refresh_token
+        if not token:
+            return
+
+        # Проверка: токен уже истёк?
+        days = self._token_days_remaining(token)
+        if days is not None and days <= 0:
+            logger.warning(
+                "OPENLANE_REFRESH_TOKEN is EXPIRED (%.1f days ago). "
+                "Will attempt login with username/password.",
+                abs(days),
+            )
+            # Очищаем токен — fallback на login()
+            self.config.refresh_token = ""
+            return
+
+        if days is not None:
+            logger.info("refresh_token: %.1f days remaining (expires %s)",
+                        days, datetime.fromtimestamp(
+                            self._decode_jwt_exp(token), tz=timezone.utc  # type: ignore[arg-type]
+                        ).strftime("%Y-%m-%d %H:%M UTC"))
+            if days < _TOKEN_REFRESH_THRESHOLD_DAYS:
+                logger.warning(
+                    "refresh_token expires in %.1f days! "
+                    "Will try to re-login after auth to get a fresh one.",
+                    days,
+                )
+
+        context.add_cookies([
+            {
+                "name": "refresh_token",
+                "value": token,
+                "domain": ".openlane.com",
+                "path": "/",
+                "httpOnly": True,
+                "secure": True,
+                "sameSite": "None",
+            }
+        ])
+        logger.info("Injected OPENLANE_REFRESH_TOKEN cookie into browser context")
+
+    # ------------------------------------------------------------------
+    # Инспекция и ротация токенов
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _decode_jwt_exp(token: str) -> int | None:
+        # Декодирование JWT exp (без проверки подписи).
+        try:
+            parts = token.split(".")
+            if len(parts) < 2:
+                return None
+            payload_b64 = parts[1]
+            # Дополнение base64 padding.
+            payload_b64 += "=" * (-len(payload_b64) % 4)
+            payload = json.loads(base64.urlsafe_b64decode(payload_b64))
+            exp = payload.get("exp")
+            return int(exp) if exp is not None else None
+        except Exception:
+            return None
+
+    @staticmethod
+    def _decode_jwt_payload(token: str) -> dict[str, Any]:
+        # Декодирование JWT payload (без проверки подписи).
+        try:
+            parts = token.split(".")
+            if len(parts) < 2:
+                return {}
+            payload_b64 = parts[1]
+            payload_b64 += "=" * (-len(payload_b64) % 4)
+            payload = json.loads(base64.urlsafe_b64decode(payload_b64))
+            return payload if isinstance(payload, dict) else {}
+        except Exception:
+            return {}
+
+    @staticmethod
+    def _token_days_remaining(token: str) -> float | None:
+        # Дней до истечения токена.
+        exp = OpenLaneAuthenticator._decode_jwt_exp(token)
+        if exp is None:
+            return None
+        now = datetime.now(timezone.utc).timestamp()
+        return (exp - now) / 86400
+
+    @staticmethod
+    def _token_seconds_remaining(token: str) -> int | None:
+        # Секунд до истечения токена.
+        exp = OpenLaneAuthenticator._decode_jwt_exp(token)
+        if exp is None:
+            return None
+        return int(exp - time.time())
+
+    @staticmethod
+    def _extract_cookie_value(context: BrowserContext, name: str) -> str | None:
+        # Извлекаем значение cookie по имени.
+        cookies = context.cookies("https://app.openlane.com")
+        for cookie in cookies:
+            if cookie.get("name") == name:
+                value = str(cookie.get("value") or "").strip()
+                if value:
+                    return value
+        return None
+
+    def _is_access_token_fresh(self, token: str) -> bool:
+        # Проверяем что access_token ещё жив.
+        remain = self._token_seconds_remaining(token)
+        return remain is not None and remain > _ACCESS_TOKEN_MIN_TTL_SECONDS
+
+    def _extract_refresh_token_from_cookies(self, context: BrowserContext) -> str | None:
+        # Извлечение refresh_token из cookies.
+        return self._extract_cookie_value(context, "refresh_token")
+
+    def _extract_access_token_from_cookies(self, context: BrowserContext) -> str | None:
+        # Извлечение access_token из cookies.
+        return self._extract_cookie_value(context, "access_token")
+
+    def _refresh_access_token_via_okta(self, refresh_token: str) -> tuple[str | None, str | None]:
+        # Прямой refresh через Okta token endpoint.
+        if not refresh_token:
+            return None, None
+        if not self.config.okta_client_id:
+            logger.warning("OPENLANE_OKTA_CLIENT_ID is empty, direct refresh disabled")
+            return None, None
+
+        form_data = urlencode(
+            {
+                "grant_type": "refresh_token",
+                "client_id": self.config.okta_client_id,
+                "redirect_uri": self.config.okta_redirect_uri,
+                "refresh_token": refresh_token,
+            }
+        ).encode("utf-8")
+        req = Request(
+            self.config.okta_token_endpoint,
+            data=form_data,
+            method="POST",
+            headers={
+                "Content-Type": "application/x-www-form-urlencoded",
+                "Accept": "application/json",
+            },
+        )
+
+        try:
+            with urlopen(req, timeout=self.config.okta_timeout_seconds) as resp:
+                payload = json.loads(resp.read().decode("utf-8"))
+            access_token = str(payload.get("access_token") or "").strip()
+            new_refresh_token = str(payload.get("refresh_token") or "").strip() or refresh_token
+            if not access_token:
+                logger.warning("Okta refresh response has no access_token")
+                return None, None
+            logger.info("Okta direct refresh succeeded")
+            return access_token, new_refresh_token
+        except HTTPError as exc:
+            try:
+                body = exc.read().decode("utf-8", errors="ignore")[:300]
+            except Exception:
+                body = ""
+            logger.warning("Okta direct refresh failed: HTTP %s %s", exc.code, body)
+            return None, None
+        except URLError as exc:
+            logger.warning("Okta direct refresh failed: %s", exc)
+            return None, None
+        except Exception as exc:
+            logger.warning("Okta direct refresh failed: %s", exc)
+            return None, None
+
+    def _persist_access_token_to_storage_state(self, context: BrowserContext, access_token: str) -> None:
+        # Сохраняем access_token cookie в контекст.
+        expires = self._decode_jwt_exp(access_token)
+        if expires is None:
+            expires = int(time.time()) + 7200
+        context.add_cookies(
+            [
+                {
+                    "name": "access_token",
+                    "value": access_token,
+                    "domain": ".openlane.com",
+                    "path": "/",
+                    "httpOnly": True,
+                    "secure": True,
+                    "sameSite": "None",
+                    "expires": int(expires),
+                }
+            ]
+        )
+
+    def _maybe_rotate_token(self, context: BrowserContext) -> None:
+        # Проверка и ротация refresh_token.
+        new_token = self._extract_refresh_token_from_cookies(context)
+        if not new_token:
+            logger.debug("No refresh_token cookie found after auth — nothing to rotate")
+            return
+
+        old_token = self.config.refresh_token or ""
+
+        # Проверяем изменился ли токен.
+        if new_token != old_token:
+            days = self._token_days_remaining(new_token)
+            logger.info(
+                "refresh_token CHANGED (new exp: %.1f days). Persisting to .env",
+                days if days is not None else -1,
+            )
+            self.config.refresh_token = new_token
+            self._persist_token_to_dotenv(new_token)
+            return
+
+        # Токен не изменился — проверяем срок.
+        days = self._token_days_remaining(new_token)
+        if days is not None:
+            logger.info("refresh_token unchanged, %.1f days remaining", days)
+            if days < _TOKEN_REFRESH_THRESHOLD_DAYS:
+                logger.warning(
+                    "⚠ refresh_token expires in %.1f days! "
+                    "Run `openlane-login` or set OPENLANE_USERNAME + OPENLANE_PASSWORD "
+                    "to auto-renew on next sync.",
+                    days,
+                )
+
+    @staticmethod
+    def _persist_token_to_dotenv(token: str) -> None:
+        # Запись refresh_token в .env.
+        env_path = Path(".env")
+        if not env_path.exists():
+            # Создаём .env с токеном.
+            env_path.write_text(
+                f"OPENLANE_REFRESH_TOKEN={token}\n",
+                encoding="utf-8",
+            )
+            logger.info("Created .env with rotated OPENLANE_REFRESH_TOKEN")
+            return
+
+        content = env_path.read_text(encoding="utf-8")
+        pattern = re.compile(r"^OPENLANE_REFRESH_TOKEN=.*$", re.MULTILINE)
+        replacement = f"OPENLANE_REFRESH_TOKEN={token}"
+
+        if pattern.search(content):
+            new_content = pattern.sub(replacement, content)
+        else:
+            new_content = content.rstrip("\n") + f"\n{replacement}\n"
+
+        env_path.write_text(new_content, encoding="utf-8")
+        # Обновляем env процесса.
+        os.environ["OPENLANE_REFRESH_TOKEN"] = token
+        logger.info("Persisted rotated OPENLANE_REFRESH_TOKEN to .env")
+
+    def bootstrap_authenticated_context(self, context: BrowserContext) -> Page:
+        # Восстановление сессии: storage_state → login → ошибка.
+        storage_state_path = Path(self.config.storage_state_file)
+
+        if storage_state_path.exists():
+            logger.info("OpenLane: restoring session from %s", storage_state_path)
+            page = context.new_page()
+            page.set_default_timeout(self.config.request_timeout_ms)
+            # Блокируем трекинг/картинки — нужен только fetch().
+            BrowserFactory.enable_resource_blocking(page)
+
+            # Проверка cookies без сетевых запросов.
+            cookies = context.cookies("https://app.openlane.com")
+            cookie_names = {c["name"] for c in cookies}
+            access_token = self._extract_access_token_from_cookies(context)
+            if access_token and self._is_access_token_fresh(access_token):
+                logger.info(
+                    "OpenLane session restored: access_token cookie present (%d cookies total)",
+                    len(cookies),
+                )
+                # Лёгкий API endpoint для установки origin в браузере.
+                page.goto(
+                    "https://app.openlane.com/api/_next/time",
+                    wait_until="domcontentloaded",
+                )
+                return page
+
+            # Пробуем прямой refresh access_token без навигации.
+            refresh_token = self._extract_refresh_token_from_cookies(context) or self.config.refresh_token
+            if refresh_token:
+                new_access_token, new_refresh_token = self._refresh_access_token_via_okta(refresh_token)
+                if new_access_token:
+                    self._persist_access_token_to_storage_state(context, new_access_token)
+                    if new_refresh_token and new_refresh_token != (self.config.refresh_token or ""):
+                        self.config.refresh_token = new_refresh_token
+                        self._persist_token_to_dotenv(new_refresh_token)
+                    self._persist_storage_state(context)
+                    page.goto(
+                        "https://app.openlane.com/api/_next/time",
+                        wait_until="domcontentloaded",
+                    )
+                    logger.info("OpenLane session restored via direct Okta refresh")
+                    return page
+                logger.warning("Direct refresh failed, trying sign_in fallback")
+
+                # Если refresh_token был только в env, добавляем cookie вручную.
+                if "refresh_token" not in cookie_names and self.config.refresh_token:
+                    self._inject_refresh_token(context)
+                    cookies = context.cookies("https://app.openlane.com")
+                    cookie_names = {c["name"] for c in cookies}
+
+            logger.warning("storage_state exists but no access_token cookie — trying navigation")
+            # Fallback: SPA обменяет refresh_token на access_token.
+            if "refresh_token" in cookie_names:
+                try:
+                    page.goto(self.config.sign_in_url, wait_until="domcontentloaded")
+                    self._wait_for_authenticated_session(page)
+                    self._persist_storage_state(context)
+                    self._maybe_rotate_token(context)
+                    return page
+                except OpenLaneAuthError:
+                    logger.warning("SPA refresh_token exchange failed")
+            page.close()
+
+        # Нет storage_state — пробуем логин.
+        if self.config.username and self.config.password:
+            return self.login(context)
+
+        raise OpenLaneAuthError(
+            "No valid session found. Either:\n"
+            "  1. Run `python scripts/explore_site.py` to login manually and save storage_state, or\n"
+            "  2. Set OPENLANE_USERNAME and OPENLANE_PASSWORD in .env"
+        )
+
+    def login(self, context: BrowserContext) -> Page:
+        self.validate_credentials()
+        page = context.new_page()
+        page.set_default_timeout(self.config.request_timeout_ms)
+        logger.info("OpenLane login: opening sign-in page %s", self.config.sign_in_url)
+        page.goto(self.config.sign_in_url, wait_until="domcontentloaded")
+        self._fill_login_form(page)
+        self._wait_for_authenticated_session(page)
+        self._persist_storage_state(context)
+        self._maybe_rotate_token(context)
+        logger.info("OpenLane login successful")
+        return page
+
+    def _fill_login_form(self, page: Page) -> None:
+        email_selectors = [
+            'input[name="username"]',
+            'input[name="user[login]"]',
+            'input[name="email"]',
+            'input[name="user[email]"]',
+            'input[autocomplete="username"]',
+            'input[type="email"]',
+            'input[type="text"]',
+            '#email',
+        ]
+        password_selectors = [
+            'input[name="password"]',
+            'input[name="user[password]"]',
+            'input[type="password"]',
+            '#password',
+        ]
+        submit_selectors = [
+            'button[type="submit"]',
+            'input[type="submit"]',
+            'button:has-text("Sign in")',
+            'button:has-text("Log in")',
+        ]
+
+        email_filled = self._fill_first(page, email_selectors, self.config.username)
+        password_filled = self._fill_first(page, password_selectors, self.config.password)
+        if not email_filled or not password_filled:
+            raise OpenLaneAuthError("OpenLane login form fields were not found")
+
+        if not self._click_first(page, submit_selectors):
+            raise OpenLaneAuthError("OpenLane login submit button was not found")
+
+    def _wait_for_authenticated_session(self, page: Page) -> None:
+        # Ждём пока SPA обменяет refresh_token → access_token.
+        try:
+            page.wait_for_url(
+                lambda url: "/sign_in" not in url.lower(),
+                timeout=min(self.config.request_timeout_ms, 30000),
+            )
+        except PlaywrightTimeoutError:
+            raise OpenLaneAuthError(
+                "OpenLane authentication failed: page stayed on sign_in "
+                "(refresh_token likely invalid or expired)"
+            )
+
+        # Проверяем что не на странице ошибки.
+        current_url = page.url.lower()
+        logger.debug("After auth redirect, URL: %s", page.url)
+        if "/sign_in" in current_url:
+            raise OpenLaneAuthError("OpenLane authentication failed: still on sign_in page")
+
+        # Ждём появления access_token cookie.
+        for attempt in range(15):
+            cookies = page.context.cookies("https://app.openlane.com")
+            cookie_names = {c["name"] for c in cookies}
+            if "access_token" in cookie_names:
+                logger.info("OpenLane authenticated: access_token cookie present (attempt %d)", attempt)
+                return
+            page.wait_for_timeout(500)
+
+        # Нет access_token cookie, но URL не sign_in — продолжаем.
+        logger.warning(
+            "access_token cookie not found after redirect, but page is at %s — continuing",
+            page.url,
+        )
+
+    def _persist_storage_state(self, context: BrowserContext) -> None:
+        if not self.config.persist_storage_state:
+            return
+        storage_state_path = Path(self.config.storage_state_file)
+        storage_state_path.parent.mkdir(parents=True, exist_ok=True)
+        context.storage_state(path=str(storage_state_path))
+        logger.info("OpenLane storage state saved to %s", storage_state_path)
+
+    @staticmethod
+    def _fill_first(page: Page, selectors: list[str], value: str) -> bool:
+        for selector in selectors:
+            locator = page.locator(selector)
+            if locator.count() == 0:
+                continue
+            try:
+                locator.first.fill(value)
+                return True
+            except PlaywrightTimeoutError:
+                continue
+        return False
+
+    @staticmethod
+    def _click_first(page: Page, selectors: list[str]) -> bool:
+        for selector in selectors:
+            locator = page.locator(selector)
+            if locator.count() == 0:
+                continue
+            try:
+                locator.first.click()
+                return True
+            except PlaywrightTimeoutError:
+                continue
+        return False

openlane_scraper/openlane/checkpoint.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+
+@dataclass(slots=True)
+class OpenLaneCheckpoint:
+    max_pages: int
+    completed_pages: list[int] = field(default_factory=list)
+    failed_pages: list[int] = field(default_factory=list)
+    total_records: int = 0
+    last_saved_at: str | None = None
+
+    @property
+    def completed_set(self) -> set[int]:
+        return set(self.completed_pages)
+
+
+class OpenLaneCheckpointStore:
+    def __init__(self, path: str | Path) -> None:
+        self.path = Path(path)
+
+    def load(self, max_pages: int) -> OpenLaneCheckpoint:
+        if not self.path.exists():
+            return OpenLaneCheckpoint(max_pages=max_pages)
+        data = json.loads(self.path.read_text(encoding="utf-8"))
+        return OpenLaneCheckpoint(
+            max_pages=int(data.get("max_pages") or max_pages),
+            completed_pages=sorted({int(page) for page in data.get("completed_pages", [])}),
+            failed_pages=sorted({int(page) for page in data.get("failed_pages", [])}),
+            total_records=int(data.get("total_records") or 0),
+            last_saved_at=data.get("last_saved_at"),
+        )
+
+    def save(self, checkpoint: OpenLaneCheckpoint) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        payload = asdict(checkpoint)
+        self.path.write_text(
+            json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
+            encoding="utf-8",
+        )

openlane_scraper/openlane/client.py

@@ -0,0 +1,376 @@
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import random
+import time
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlencode
+
+from playwright.sync_api import Page
+
+from ..core.config import OpenLaneConfig
+
+logger = logging.getLogger("openlane_scraper.openlane.client")
+
+
+class OpenLaneRequestError(RuntimeError):
+    def __init__(self, message: str, *, status_code: int | None = None) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+
+
+@dataclass(slots=True)
+class OpenLanePageResult:
+    page: int
+    records: list[dict[str, Any]]
+    raw_payload: dict[str, Any]
+    status_code: int
+
+
+def _decode_access_token(token: str) -> dict[str, Any]:
+    # Декодирование JWT payload (без проверки подписи).
+    parts = token.split(".")
+    if len(parts) < 2:
+        return {}
+    payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4)
+    try:
+        return json.loads(base64.urlsafe_b64decode(payload_b64))
+    except Exception:
+        return {}
+
+
+class OpenLaneClient:
+    # HTTP-клиент OpenLane search API через Playwright.
+
+    def __init__(self, authenticated_page: Page, config: OpenLaneConfig) -> None:
+        self.page = authenticated_page
+        self.config = config
+        self._user_id: str | None = None
+        self._dealership_id: str | None = None
+        self._init_ids_from_cookies()
+
+    def _init_ids_from_cookies(self) -> None:
+        # Извлекаем user_id и dealership_id из access_token.
+        cookies = self.page.context.cookies("https://app.openlane.com")
+        for cookie in cookies:
+            if cookie.get("name") == "access_token":
+                payload = _decode_access_token(cookie["value"])
+                self._user_id = str(payload.get("user_id", ""))
+                on_behalf = payload.get("on_behalf_of", {})
+                self._dealership_id = str(on_behalf.get("dealership_id", ""))
+                if self._user_id and self._dealership_id:
+                    logger.info(
+                        "OpenLane client: user_id=%s dealership_id=%s",
+                        self._user_id, self._dealership_id,
+                    )
+                return
+        logger.warning("OpenLane client: access_token cookie not found, API requests may fail")
+
+    def _build_headers(self) -> dict[str, str]:
+        # Заголовки как у фронтенда OpenLane.
+        headers: dict[str, str] = {
+            "Accept": "application/json, application/vnd.backlotcars.v3",
+            "application": "webapp",
+        }
+        if self._user_id:
+            headers["x-rbz-user-id"] = self._user_id
+        if self._dealership_id:
+            headers["x-rbz-dealership-id"] = self._dealership_id
+        return headers
+
+    def fetch_page(self, page: int) -> OpenLanePageResult:
+        params = {
+            "page": page,
+            "source_tab": self.config.source_tab,
+            "sale_types": self.config.sale_types,
+        }
+        if self.config.page_size > 0:
+            params["per_page"] = self.config.page_size
+
+        self._sleep_with_jitter()
+        url = f"{self.config.api_search_url}?{urlencode(params)}"
+        logger.debug("OpenLane fetch page=%s url=%s", page, url)
+
+        # fetch() с авторизацией в браузере.
+        headers_js = json.dumps(self._build_headers())
+        fetch_result = self.page.evaluate(
+            """async ([url, headersJson]) => {
+                const headers = JSON.parse(headersJson);
+                const resp = await fetch(url, {
+                    method: 'GET',
+                    credentials: 'include',
+                    headers: headers
+                });
+                const text = await resp.text();
+                return { status: resp.status, body: text };
+            }""",
+            [url, headers_js],
+        )
+        return self._parse_response(page, fetch_result)
+
+    def fetch_vehicle_images(self, vehicle_id: str | int) -> list[dict[str, Any]]:
+        """Загружает изображения автомобиля через `/api/vehicles/{id}/images`."""
+        if vehicle_id is None:
+            return []
+
+        vehicle_id_str = str(vehicle_id).strip()
+        if not vehicle_id_str:
+            return []
+
+        self._sleep_with_jitter()
+        url = f"https://app.openlane.com/api/vehicles/{vehicle_id_str}/images"
+        logger.debug("OpenLane fetch images vehicle_id=%s", vehicle_id_str)
+
+        headers_js = json.dumps(self._build_headers())
+        fetch_result = self.page.evaluate(
+            """async ([url, headersJson]) => {
+                const headers = JSON.parse(headersJson);
+                const resp = await fetch(url, {
+                    method: 'GET',
+                    credentials: 'include',
+                    headers: headers
+                });
+                const text = await resp.text();
+                return { status: resp.status, body: text };
+            }""",
+            [url, headers_js],
+        )
+
+        status_code = int(fetch_result.get("status", 0))
+        text = str(fetch_result.get("body", ""))
+        if status_code == 404:
+            return []
+        if status_code == 403:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
+                status_code=status_code,
+            )
+        if status_code == 429:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
+                status_code=status_code,
+            )
+        if status_code >= 500:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
+                status_code=status_code,
+            )
+        if status_code == 401:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
+                status_code=status_code,
+            )
+        if status_code >= 400:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
+                status_code=status_code,
+            )
+
+        if not text.strip():
+            return []
+
+        try:
+            payload = json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
+            ) from exc
+
+        return self._extract_vehicle_images(payload)
+
+    def fetch_vehicle_images_batch(self, vehicle_ids: list[str | int]) -> dict[str, list[dict[str, Any]]]:
+        """Пакетно загружает изображения автомобилей через `/api/vehicles/{id}/images`.
+
+        Возвращает словарь `vehicle_id -> список image-объектов`.
+        """
+        normalized_ids: list[str] = []
+        seen: set[str] = set()
+        for vehicle_id in vehicle_ids:
+            vehicle_id_str = str(vehicle_id).strip()
+            if not vehicle_id_str or vehicle_id_str in seen:
+                continue
+            seen.add(vehicle_id_str)
+            normalized_ids.append(vehicle_id_str)
+
+        if not normalized_ids:
+            return {}
+
+        self._sleep_with_jitter()
+        headers_js = json.dumps(self._build_headers())
+        fetch_results = self.page.evaluate(
+            """async ([vehicleIds, headersJson]) => {
+                const headers = JSON.parse(headersJson);
+                const tasks = vehicleIds.map(async (vehicleId) => {
+                    try {
+                        const url = `https://app.openlane.com/api/vehicles/${vehicleId}/images`;
+                        const resp = await fetch(url, {
+                            method: 'GET',
+                            credentials: 'include',
+                            headers: headers,
+                        });
+                        const text = await resp.text();
+                        return { vehicleId, status: resp.status, body: text };
+                    } catch (error) {
+                        return { vehicleId, status: 0, body: '', error: String(error) };
+                    }
+                });
+                return await Promise.all(tasks);
+            }""",
+            [normalized_ids, headers_js],
+        )
+
+        result_map: dict[str, list[dict[str, Any]]] = {}
+        for item in fetch_results:
+            vehicle_id_str = str(item.get("vehicleId", "")).strip()
+            status_code = int(item.get("status", 0))
+            text = str(item.get("body", ""))
+
+            if not vehicle_id_str:
+                continue
+
+            if status_code == 404:
+                result_map[vehicle_id_str] = []
+                continue
+            if status_code == 403:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
+                    status_code=status_code,
+                )
+            if status_code == 429:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
+                    status_code=status_code,
+                )
+            if status_code >= 500 or status_code == 0:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
+                    status_code=status_code if status_code else 500,
+                )
+            if status_code == 401:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
+                    status_code=status_code,
+                )
+            if status_code >= 400:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
+                    status_code=status_code,
+                )
+
+            if not text.strip():
+                result_map[vehicle_id_str] = []
+                continue
+
+            try:
+                payload = json.loads(text)
+            except json.JSONDecodeError as exc:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
+                ) from exc
+
+            result_map[vehicle_id_str] = self._extract_vehicle_images(payload)
+
+        return result_map
+
+    def _parse_response(self, page: int, fetch_result: dict[str, Any]) -> OpenLanePageResult:
+        # Парсинг ответа fetch(): {status, body}.
+        status_code = int(fetch_result.get("status", 0))
+        text = str(fetch_result.get("body", ""))
+        logger.debug(
+            "OpenLane response: page=%s status=%s body_len=%d body_preview=%.500s",
+            page, status_code, len(text), text[:500],
+        )
+        if status_code == 403:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned 403 Forbidden — possible block, stopping",
+                status_code=status_code,
+            )
+        if status_code == 429:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned 429 Too Many Requests — rate limited",
+                status_code=status_code,
+            )
+        if status_code >= 500:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned server error {status_code}",
+                status_code=status_code,
+            )
+        if status_code == 401:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned 401; session is not authenticated",
+                status_code=status_code,
+            )
+        if status_code >= 400:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned unexpected status {status_code}",
+                status_code=status_code,
+            )
+
+        try:
+            payload = json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise OpenLaneRequestError(f"OpenLane page={page} returned invalid JSON") from exc
+
+        records = self._extract_records(payload)
+        logger.debug(
+            "OpenLane extracted: page=%s records=%d top_keys=%s",
+            page, len(records), list(payload.keys())[:10],
+        )
+        return OpenLanePageResult(
+            page=page,
+            records=records,
+            raw_payload=payload,
+            status_code=status_code,
+        )
+
+    @staticmethod
+    def _extract_records(payload: dict[str, Any]) -> list[dict[str, Any]]:
+        candidates = [
+            payload.get("results"),
+            payload.get("items"),
+            payload.get("data"),
+            payload.get("vehicles"),
+        ]
+        for candidate in candidates:
+            if isinstance(candidate, list):
+                return [item for item in candidate if isinstance(item, dict)]
+            if isinstance(candidate, dict):
+                nested_candidates = [
+                    candidate.get("results"),
+                    candidate.get("items"),
+                    candidate.get("vehicles"),
+                ]
+                for nested in nested_candidates:
+                    if isinstance(nested, list):
+                        return [item for item in nested if isinstance(item, dict)]
+        return []
+
+    @staticmethod
+    def _extract_vehicle_images(payload: Any) -> list[dict[str, Any]]:
+        candidates: list[Any] = []
+        if isinstance(payload, dict):
+            candidates.extend(
+                [
+                    payload.get("vehicle_images"),
+                    payload.get("images"),
+                    payload.get("data"),
+                ]
+            )
+            data = payload.get("data")
+            if isinstance(data, dict):
+                candidates.extend([data.get("vehicle_images"), data.get("images")])
+        elif isinstance(payload, list):
+            candidates.append(payload)
+
+        for candidate in candidates:
+            if isinstance(candidate, list):
+                return [item for item in candidate if isinstance(item, dict)]
+        return []
+
+    def _sleep_with_jitter(self) -> None:
+        low = min(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
+        high = max(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
+        time.sleep(random.uniform(low, high))

openlane_scraper/openlane/mapper.py

@@ -0,0 +1,424 @@
+"""Маппер: JSON-запись OpenLane API → CarRecord для сохранения в БД."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from datetime import datetime, timezone
+from typing import Any
+
+from ..storage.schemas import CarRecord, ImageRecord
+
+logger = logging.getLogger("openlane_scraper.openlane.mapper")
+
+# Маппинг body_type из OpenLane в наш enum.
+BODY_MAP: dict[str, str] = {
+    "sedan": "SEDAN",
+    "coupe": "COUPE",
+    "suv": "SUV",
+    "sport utility": "SUV",
+    "crossover": "SUV",
+    "hatchback": "HATCHBACK",
+    "minivan": "MINIVAN",
+    "van": "MINIVAN",
+    "wagon": "STATION_WAGON",
+    "station wagon": "STATION_WAGON",
+    "pickup": "PICKUP",
+    "truck": "TRUCK",
+    "convertible": "OPEN",
+    "cabriolet": "OPEN",
+    "roadster": "OPEN",
+    "rv": "RV",
+    "motorhome": "RV",
+}
+
+DRIVE_MAP: dict[str, str] = {
+    "fwd": "FWD",
+    "front wheel drive": "FWD",
+    "front-wheel drive": "FWD",
+    "rwd": "RWD",
+    "rear wheel drive": "RWD",
+    "rear-wheel drive": "RWD",
+    "awd": "4WD",
+    "4wd": "4WD",
+    "all wheel drive": "4WD",
+    "all-wheel drive": "4WD",
+    "4x4": "4WD",
+    "2wd": "2WD",
+    "two wheel drive": "2WD",
+}
+
+GEARBOX_MAP: dict[str, str] = {
+    "automatic": "AT",
+    "auto": "AT",
+    "at": "AT",
+    "manual": "MT",
+    "mt": "MT",
+    "cvt": "CVT",
+    "continuously variable": "CVT",
+    "electric": "EV",
+    "ev": "EV",
+}
+
+_MILEAGE_RE = re.compile(r"[\d,]+")
+_ENGINE_RE = re.compile(r"(\d+\.?\d*)\s*[lL]")
+
+
+def _safe_str(value: Any, default: str = "") -> str:
+    if value is None:
+        return default
+    return str(value).strip() or default
+
+
+def _safe_int(value: Any) -> int | None:
+    if value is None:
+        return None
+    try:
+        cleaned = str(value).replace(",", "").strip()
+        if not cleaned:
+            return None
+        return int(float(cleaned))
+    except (ValueError, TypeError):
+        return None
+
+
+def _normalize_enum(raw: Any, mapping: dict[str, str], default: str = "NA") -> str:
+    if not raw:
+        return default
+    key = str(raw).strip().lower()
+    return mapping.get(key, default)
+
+
+def _generate_parser_id(origin_id: str) -> str:
+    return hashlib.sha256(origin_id.encode()).hexdigest()[:40]
+
+
+def _generate_slug(year: int | None, brand: str, model: str, origin_id: str) -> str:
+    parts = []
+    if year:
+        parts.append(str(year))
+    parts.append(brand.lower())
+    parts.append(model.lower())
+    parts.append(origin_id.replace(":", "-"))
+    slug = "-".join(parts)
+    slug = re.sub(r"[^a-z0-9\-]", "-", slug)
+    slug = re.sub(r"-+", "-", slug).strip("-")
+    return slug[:200]
+
+
+def _build_openlane_origin_id(raw_id: Any) -> str | None:
+    """Строит origin_id в формате openlane:id."""
+    normalized = _safe_str(raw_id)
+    if not normalized:
+        return None
+    if normalized.lower().startswith("openlane:"):
+        normalized = _safe_str(normalized.split(":", 1)[1])
+    if not normalized:
+        return None
+    return f"openlane:{normalized}"
+
+
+def _extract_nested(record: dict[str, Any], *keys: str) -> Any:
+    """Извлекает значение из вложенного словаря по цепочке ключей."""
+    obj: Any = record
+    for key in keys:
+        if isinstance(obj, dict):
+            obj = obj.get(key)
+        else:
+            return None
+    return obj
+
+
+def _extract_images(record: dict[str, Any]) -> list[ImageRecord]:
+    """Извлекает изображения из записи OpenLane."""
+    images: list[ImageRecord] = []
+    seen_urls: set[str] = set()
+
+    raw_images = (
+        record.get("images")
+        or record.get("photos")
+        or record.get("media", {}).get("images")
+        or record.get("image_urls")
+        or []
+    )
+
+    if isinstance(raw_images, list):
+        for idx, img in enumerate(raw_images):
+            if isinstance(img, str):
+                url = img.strip()
+                if url and url not in seen_urls:
+                    seen_urls.add(url)
+                    images.append(ImageRecord(
+                        fullres_image=url,
+                        preview_image=url,
+                        order_index=idx,
+                    ))
+            elif isinstance(img, dict):
+                fullres = _safe_str(
+                    img.get("full") or img.get("fullres") or img.get("url")
+                    or img.get("original") or img.get("large") or img.get("href")
+                    or img.get("large_resolution_url")
+                )
+                preview = _safe_str(
+                    img.get("thumbnail") or img.get("thumb") or img.get("preview")
+                    or img.get("small") or img.get("low_resolution_url")
+                    or fullres
+                )
+                if fullres and fullres not in seen_urls:
+                    seen_urls.add(fullres)
+                    images.append(ImageRecord(
+                        fullres_image=fullres,
+                        preview_image=preview,
+                        order_index=idx,
+                    ))
+
+    # Fallback: одиночное изображение.
+    if not images:
+        single = (
+            record.get("image_url")
+            or record.get("image")
+            or record.get("large_resolution_url")
+            or record.get("low_resolution_url")
+            or record.get("thumbnail")
+            or record.get("photo_url")
+            or _extract_nested(record, "media", "primary")
+        )
+        if single and isinstance(single, str) and single.strip():
+            images.append(ImageRecord(
+                fullres_image=single.strip(),
+                preview_image=single.strip(),
+                order_index=0,
+            ))
+
+    return images
+
+
+def _parse_mileage(raw: Any) -> int:
+    if raw is None:
+        return 0
+    if isinstance(raw, (int, float)):
+        return max(0, int(raw))
+    text = str(raw)
+    match = _MILEAGE_RE.search(text)
+    if match:
+        try:
+            return max(0, int(match.group().replace(",", "")))
+        except ValueError:
+            pass
+    return 0
+
+
+def _parse_engine_volume_cc(raw: Any) -> int | None:
+    """Парсит объём двигателя и возвращает значение в кубических сантиметрах."""
+    if raw is None:
+        return None
+    if isinstance(raw, (int, float)):
+        value = float(raw)
+        # Если значение < 20 — скорее всего это литры, конвертируем в cc.
+        if 0 < value < 20:
+            return int(value * 1000)
+        if value >= 100:
+            return int(value)
+        return None
+    text = str(raw)
+    match = _ENGINE_RE.search(text)
+    if match:
+        liters = float(match.group(1))
+        return int(liters * 1000)
+    return None
+
+
+def map_openlane_record(record: dict[str, Any]) -> CarRecord | None:
+    """Маппит одну запись из OpenLane API в CarRecord.
+
+    Возвращает None, если запись не содержит минимально необходимых данных.
+    """
+    # Извлекаем идентификатор.
+    raw_id = (
+        record.get("id")
+        or record.get("vehicle_id")
+        or record.get("listing_id")
+        or record.get("vin")
+    )
+    if not raw_id:
+        logger.debug("Skipping record without id: %s", record.get("vin", "unknown"))
+        return None
+
+    origin_id = _build_openlane_origin_id(raw_id)
+    if not origin_id:
+        logger.debug("Skipping record with malformed id: %s", raw_id)
+        return None
+
+    # Бренд и модель — обязательные поля.
+    brand = _safe_str(
+        record.get("make")
+        or record.get("brand")
+        or record.get("manufacturer")
+        or _extract_nested(record, "vehicle", "make")
+    ).upper()
+
+    model = _safe_str(
+        record.get("model")
+        or record.get("model_name")
+        or _extract_nested(record, "vehicle", "model")
+    ).upper()
+
+    if not brand or not model:
+        logger.debug("Skipping record without brand/model: %s", origin_id)
+        return None
+
+    year = _safe_int(
+        record.get("year")
+        or record.get("model_year")
+        or _extract_nested(record, "vehicle", "year")
+    )
+
+    price = _safe_int(
+        record.get("price")
+        or record.get("current_bid")
+        or record.get("buy_now_price")
+        or record.get("asking_price")
+        or record.get("sale_price")
+        or _extract_nested(record, "pricing", "current")
+        or _extract_nested(record, "pricing", "buy_now")
+    )
+
+    currency = _safe_str(
+        record.get("currency")
+        or record.get("currency_code")
+        or _extract_nested(record, "pricing", "currency"),
+        "USD",
+    ).upper()
+    if currency not in {"JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD"}:
+        currency = "USD"
+
+    mileage = _parse_mileage(
+        record.get("mileage")
+        or record.get("odometer")
+        or record.get("odometer_reading")
+        or _extract_nested(record, "vehicle", "mileage")
+    )
+
+    color = _safe_str(
+        record.get("color")
+        or record.get("exterior_color")
+        or _extract_nested(record, "vehicle", "color"),
+        "other",
+    ).lower()
+
+    body_type = _normalize_enum(
+        record.get("body_type")
+        or record.get("body_style")
+        or record.get("vehicle_type")
+        or _extract_nested(record, "vehicle", "body_type"),
+        BODY_MAP,
+        "OTHER",
+    )
+
+    drive = _normalize_enum(
+        record.get("drive_type")
+        or record.get("drivetrain")
+        or record.get("drive")
+        or _extract_nested(record, "vehicle", "drivetrain"),
+        DRIVE_MAP,
+    )
+
+    gearbox = _normalize_enum(
+        record.get("transmission")
+        or record.get("gearbox")
+        or _extract_nested(record, "vehicle", "transmission"),
+        GEARBOX_MAP,
+    )
+
+    engine_volume = _parse_engine_volume_cc(
+        record.get("engine")
+        or record.get("engine_size")
+        or record.get("displacement")
+        or _extract_nested(record, "vehicle", "engine")
+    )
+
+    # URL записи на OpenLane.
+    origin_url = _safe_str(
+        record.get("url")
+        or record.get("listing_url")
+        or record.get("detail_url")
+        or record.get("permalink")
+    )
+    if not origin_url:
+        origin_url = f"https://app.openlane.com/vehicles/{raw_id}"
+
+    country = _safe_str(
+        record.get("country")
+        or record.get("location_country")
+        or _extract_nested(record, "location", "country"),
+        "US",
+    ).upper()
+    if country not in {"JP", "KR", "US", "CA", "NA"}:
+        country = "US"
+
+    is_damaged = bool(
+        record.get("is_damaged")
+        or record.get("has_damage")
+        or record.get("damage_type")
+    )
+
+    vin = _safe_str(record.get("vin") or _extract_nested(record, "vehicle", "vin"))
+    evaluation = vin if vin else None
+
+    selling_type = "AUCTION"
+    sale_type = _safe_str(record.get("sale_type") or record.get("listing_type")).lower()
+    if sale_type in {"buy_now", "fixed_price", "stock"}:
+        selling_type = "STOCK"
+    elif sale_type in {"tender"}:
+        selling_type = "TENDER"
+
+    images = _extract_images(record)
+
+    parser_id = _generate_parser_id(origin_id)
+    slug = _generate_slug(year, brand, model, origin_id)
+
+    return CarRecord(
+        parser_id=parser_id,
+        brand=brand,
+        model=model,
+        year=year,
+        price=price,
+        currency=currency,
+        mileage=mileage,
+        country=country,
+        is_sold=False,
+        color=color,
+        drive=drive if drive != "NA" else None,
+        gearbox=gearbox if gearbox != "NA" else None,
+        body_type=body_type,
+        engine_volume=engine_volume,
+        selling_type=selling_type,
+        origin="OPENLANE",
+        origin_url=origin_url,
+        origin_id=origin_id,
+        is_damaged=is_damaged,
+        evaluation=evaluation,
+        slug=slug,
+        images=images,
+    )
+
+
+def map_openlane_records(records: list[dict[str, Any]]) -> list[CarRecord]:
+    """Маппит список записей OpenLane API в список CarRecord.
+
+    Пропускает записи без минимально необходимых данных.
+    """
+    result: list[CarRecord] = []
+    for record in records:
+        try:
+            car = map_openlane_record(record)
+            if car is not None:
+                result.append(car)
+        except Exception:
+            logger.warning(
+                "Failed to map OpenLane record id=%s",
+                record.get("id", "unknown"),
+                exc_info=True,
+            )
+    return result

openlane_scraper/openlane/runner.py

@@ -0,0 +1,273 @@
+from __future__ import annotations
+
+import logging
+import time
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from playwright.sync_api import sync_playwright
+
+from ..browser.factory import BrowserFactory
+from ..core.config import Settings
+from ..core.logs import set_trace_id, setup_logging
+from .auth import OpenLaneAuthenticator
+from .checkpoint import OpenLaneCheckpoint, OpenLaneCheckpointStore
+from .client import OpenLaneClient, OpenLanePageResult, OpenLaneRequestError
+from .writer import OpenLaneResultWriter
+
+logger = logging.getLogger("openlane_scraper.openlane.runner")
+
+
+@dataclass(slots=True)
+class OpenLaneScrapeResult:
+    jsonl_path: str
+    aggregated_path: str
+    checkpoint_path: str
+    storage_state_path: str
+    completed_pages: list[int]
+    failed_pages: list[int]
+    total_records: int
+    elapsed_seconds: float
+
+
+class OpenLaneScrapeRunner:
+    def __init__(self, runtime_settings: Settings | None = None) -> None:
+        self.settings = runtime_settings or Settings()
+        setup_logging(self.settings.log_level, self.settings.log_file)
+        self.trace_id = f"openlane-{uuid.uuid4().hex[:8]}"
+        set_trace_id(self.trace_id)
+        self.openlane = self.settings.openlane
+        self.browser_factory = BrowserFactory(self.settings)
+        self.authenticator = OpenLaneAuthenticator(self.openlane)
+        self.writer = OpenLaneResultWriter(
+            self.openlane.jsonl_output,
+            self.openlane.aggregated_output,
+        )
+        self.checkpoint_store = OpenLaneCheckpointStore(self.openlane.checkpoint_file)
+
+    def run(
+        self,
+        *,
+        max_pages: int | None = None,
+        concurrency: int | None = None,
+        resume: bool = False,
+        checkpoint_every_pages: int | None = None,
+    ) -> OpenLaneScrapeResult:
+        started_at = time.perf_counter()
+        max_pages = max_pages or self.openlane.max_pages
+        concurrency = max(1, min(concurrency or self.openlane.concurrency, 5))
+        checkpoint_every_pages = checkpoint_every_pages or self.openlane.checkpoint_every_pages
+
+        checkpoint = self.checkpoint_store.load(max_pages=max_pages) if resume else OpenLaneCheckpoint(max_pages=max_pages)
+        checkpoint.max_pages = max_pages
+
+        if not resume:
+            self._reset_outputs()
+
+        logger.info(
+            "Starting OpenLane scrape: max_pages=%s concurrency=%s resume=%s checkpoint_every_pages=%s",
+            max_pages,
+            concurrency,
+            resume,
+            checkpoint_every_pages,
+        )
+
+        with sync_playwright() as playwright:
+            browser = self.browser_factory.create_browser(playwright)
+            try:
+                auth_pages = []
+                for worker_index in range(concurrency):
+                    storage_state_path = self.openlane.storage_state_file if Path(self.openlane.storage_state_file).exists() else None
+                    context = self.browser_factory.create_context(browser, storage_state_path=storage_state_path)
+                    auth_page = self.authenticator.bootstrap_authenticated_context(context)
+                    auth_pages.append(auth_page)
+                    logger.info("OpenLane worker session initialized: worker=%s", worker_index + 1)
+
+                pending_pages = [
+                    page for page in range(1, max_pages + 1)
+                    if page not in checkpoint.completed_set
+                ]
+                self._run_workers(auth_pages, pending_pages, checkpoint, checkpoint_every_pages)
+            finally:
+                browser.close()
+
+        self.checkpoint_store.save(checkpoint)
+        aggregated = self.writer.finalize(
+            max_pages=max_pages,
+            completed_pages=checkpoint.completed_pages,
+            failed_pages=checkpoint.failed_pages,
+        )
+        elapsed_seconds = time.perf_counter() - started_at
+        logger.info(
+            "OpenLane scrape finished: completed_pages=%s failed_pages=%s total_records=%s elapsed=%.2fs",
+            len(set(checkpoint.completed_pages)),
+            len(set(checkpoint.failed_pages)),
+            aggregated["total_records"],
+            elapsed_seconds,
+        )
+        return OpenLaneScrapeResult(
+            jsonl_path=str(Path(self.openlane.jsonl_output)),
+            aggregated_path=str(Path(self.openlane.aggregated_output)),
+            checkpoint_path=str(Path(self.openlane.checkpoint_file)),
+            storage_state_path=str(Path(self.openlane.storage_state_file)),
+            completed_pages=sorted(set(checkpoint.completed_pages)),
+            failed_pages=sorted(set(checkpoint.failed_pages)),
+            total_records=int(aggregated["total_records"]),
+            elapsed_seconds=elapsed_seconds,
+        )
+
+    def _run_workers(
+        self,
+        auth_pages: list,
+        pending_pages: list[int],
+        checkpoint: OpenLaneCheckpoint,
+        checkpoint_every_pages: int,
+    ) -> None:
+        # Последовательная обработка страниц (sync API — однопоточный).
+        started_at = time.perf_counter()
+
+        for idx, page_num in enumerate(pending_pages):
+            worker_index = (idx % len(auth_pages)) + 1
+            auth_page = auth_pages[idx % len(auth_pages)]
+
+            try:
+                result = self._fetch_page_with_retry(auth_page, page_num, worker_index)
+                self._handle_success(result, checkpoint, started_at)
+            except Exception as exc:
+                self._handle_failure(page_num, exc, checkpoint, started_at)
+
+            processed_count = len(set(checkpoint.completed_pages)) + len(set(checkpoint.failed_pages))
+            if processed_count and processed_count % checkpoint_every_pages == 0:
+                checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat()
+                self.checkpoint_store.save(checkpoint)
+                logger.info(
+                    "OpenLane checkpoint saved: processed=%s completed=%s failed=%s",
+                    processed_count,
+                    len(set(checkpoint.completed_pages)),
+                    len(set(checkpoint.failed_pages)),
+                )
+
+    def _fetch_page_with_retry(self, authenticated_page, page: int, worker_index: int) -> OpenLanePageResult:
+        set_trace_id(f"{self.trace_id}-w{worker_index}-p{page}")
+        client = OpenLaneClient(authenticated_page, self.openlane)
+        retry_schedule = self.openlane.retry_schedule_seconds
+
+        for attempt in range(len(retry_schedule) + 1):
+            try:
+                logger.debug("OpenLane fetch attempt: page=%s worker=%s attempt=%s", page, worker_index, attempt + 1)
+                return client.fetch_page(page)
+            except OpenLaneRequestError as exc:
+                is_retryable = exc.status_code in {403, 429} or (exc.status_code is not None and exc.status_code >= 500)
+                if not is_retryable or attempt >= len(retry_schedule):
+                    logger.error(
+                        "OpenLane fetch failed permanently: page=%s worker=%s status=%s error=%s",
+                        page,
+                        worker_index,
+                        exc.status_code,
+                        exc,
+                    )
+                    raise
+                delay = retry_schedule[attempt]
+                logger.warning(
+                    "OpenLane retry scheduled: page=%s worker=%s status=%s delay=%.1fs attempt=%s",
+                    page,
+                    worker_index,
+                    exc.status_code,
+                    delay,
+                    attempt + 1,
+                )
+                time.sleep(delay)
+            except Exception:
+                if attempt >= len(retry_schedule):
+                    raise
+                delay = retry_schedule[attempt]
+                logger.warning(
+                    "OpenLane transient error retry: page=%s worker=%s delay=%.1fs attempt=%s",
+                    page,
+                    worker_index,
+                    delay,
+                    attempt + 1,
+                    exc_info=True,
+                )
+                time.sleep(delay)
+
+        raise RuntimeError(f"OpenLane page {page} exhausted retries")
+
+    def _handle_success(
+        self,
+        result: OpenLanePageResult,
+        checkpoint: OpenLaneCheckpoint,
+        started_at: float,
+    ) -> None:
+        written = self.writer.append_page(result.page, result.records)
+        checkpoint.completed_pages = sorted(set(checkpoint.completed_pages) | {result.page})
+        checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) - {result.page})
+        checkpoint.total_records += written
+        checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat()
+        self._log_progress(result.page, checkpoint, started_at, written, None)
+
+    def _handle_failure(
+        self,
+        page: int,
+        exc: Exception,
+        checkpoint: OpenLaneCheckpoint,
+        started_at: float,
+    ) -> None:
+        checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) | {page})
+        checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat()
+        self._log_progress(page, checkpoint, started_at, 0, exc)
+
+    def _log_progress(
+        self,
+        page: int,
+        checkpoint: OpenLaneCheckpoint,
+        started_at: float,
+        records_written: int,
+        exc: Exception | None,
+    ) -> None:
+        completed = len(set(checkpoint.completed_pages))
+        failed = len(set(checkpoint.failed_pages))
+        elapsed_seconds = max(time.perf_counter() - started_at, 0.001)
+        pages_per_minute = (completed + failed) / elapsed_seconds * 60.0
+        if exc is None:
+            logger.info(
+                "OpenLane progress: page=%s completed=%s failed=%s records=%s total_records=%s speed=%.2f pages/min",
+                page,
+                completed,
+                failed,
+                records_written,
+                checkpoint.total_records,
+                pages_per_minute,
+            )
+        else:
+            logger.error(
+                "OpenLane page error: page=%s completed=%s failed=%s total_records=%s speed=%.2f pages/min error=%s",
+                page,
+                completed,
+                failed,
+                checkpoint.total_records,
+                pages_per_minute,
+                exc,
+            )
+
+    def interactive_login(self) -> str:
+        setup_logging(self.settings.log_level, self.settings.log_file)
+        with sync_playwright() as playwright:
+            browser = self.browser_factory.create_browser(playwright)
+            try:
+                context = self.browser_factory.create_context(browser)
+                return self.authenticator.interactive_login_and_persist(context)
+            finally:
+                browser.close()
+
+    def _reset_outputs(self) -> None:
+        for raw_path in (
+            self.openlane.jsonl_output,
+            self.openlane.aggregated_output,
+            self.openlane.checkpoint_file,
+        ):
+            path = Path(raw_path)
+            if path.exists():
+                path.unlink()

openlane_scraper/openlane/writer.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+class OpenLaneResultWriter:
+    def __init__(self, jsonl_path: str | Path, aggregated_path: str | Path) -> None:
+        self.jsonl_path = Path(jsonl_path)
+        self.aggregated_path = Path(aggregated_path)
+
+    def ensure_parent_dirs(self) -> None:
+        self.jsonl_path.parent.mkdir(parents=True, exist_ok=True)
+        self.aggregated_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def append_page(self, page: int, records: list[dict[str, Any]]) -> int:
+        self.ensure_parent_dirs()
+        written = 0
+        with self.jsonl_path.open("a", encoding="utf-8") as fh:
+            for record in records:
+                payload = {
+                    "page": page,
+                    "record": record,
+                }
+                fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
+                written += 1
+        return written
+
+    def finalize(self, *, max_pages: int, completed_pages: list[int], failed_pages: list[int]) -> dict[str, Any]:
+        self.ensure_parent_dirs()
+        records: list[dict[str, Any]] = []
+        if self.jsonl_path.exists():
+            with self.jsonl_path.open("r", encoding="utf-8") as fh:
+                for line in fh:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    item = json.loads(line)
+                    records.append(item)
+
+        summary = {
+            "max_pages": max_pages,
+            "completed_pages": sorted(completed_pages),
+            "failed_pages": sorted(failed_pages),
+            "total_pages_completed": len(set(completed_pages)),
+            "total_pages_failed": len(set(failed_pages)),
+            "total_records": len(records),
+            "items": records,
+        }
+        self.aggregated_path.write_text(
+            json.dumps(summary, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        return summary

openlane_scraper/scraper.py

@@ -0,0 +1,514 @@
+"""Главный оркестратор скрапинга OpenLane с сохранением в БД.
+
+Использует OpenLane API для получения данных и PersistenceService для записи в PostgreSQL.
+Batched upsert, SyncRun-аудит, early-stop при отсутствии новых записей.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from playwright.sync_api import sync_playwright
+
+from .browser.factory import BrowserFactory
+from .core.config import Settings
+from .core.logs import set_trace_id, setup_logging
+from .core.runtime_config import RuntimeConfig, apply_filters, load_runtime_config
+from .openlane.auth import OpenLaneAuthenticator
+from .openlane.client import OpenLaneClient, OpenLanePageResult, OpenLaneRequestError
+from .openlane.mapper import map_openlane_records
+from .storage.db import PersistenceService
+from .storage.schemas import CarRecord, ImageRecord
+
+logger = logging.getLogger("openlane_scraper.scraper")
+
+
+class OpenLaneScraper:
+    """Оркестратор: OpenLane API → маппинг → DB upsert."""
+
+    def __init__(self, runtime_settings: Settings | None = None) -> None:
+        self.settings = runtime_settings or Settings()
+        setup_logging(self.settings.log_level, self.settings.log_file)
+        self.trace_id = f"openlane-sync-{uuid.uuid4().hex[:8]}"
+        set_trace_id(self.trace_id)
+        self.openlane_config = self.settings.openlane
+        self.browser_factory = BrowserFactory(self.settings)
+        self.authenticator = OpenLaneAuthenticator(self.openlane_config)
+        self.persistence = PersistenceService(self.settings)
+        self.runtime_config = load_runtime_config(self.settings.runtime_config_file)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        pass
+
+    @staticmethod
+    def _normalize_origin_id(raw_id: Any) -> str | None:
+        if raw_id is None:
+            return None
+        normalized = str(raw_id).strip()
+        if not normalized:
+            return None
+        if normalized.lower().startswith("openlane:"):
+            normalized = normalized.split(":", 1)[1].strip()
+        if not normalized:
+            return None
+        return f"openlane:{normalized}"
+
+    @staticmethod
+    def _build_image_records(images_payload: list[dict[str, Any]]) -> list[ImageRecord]:
+        images: list[ImageRecord] = []
+        seen: set[str] = set()
+        for idx, img in enumerate(images_payload):
+            fullres = str(
+                img.get("url")
+                or img.get("large_resolution_url")
+                or img.get("image_large")
+                or img.get("image")
+                or img.get("full")
+                or img.get("fullres")
+                or ""
+            ).strip()
+            if not fullres or fullres in seen:
+                continue
+            preview = str(
+                img.get("low_resolution_url")
+                or img.get("medium_resolution_url")
+                or img.get("thumbnail_url")
+                or img.get("thumbnail")
+                or img.get("thumb")
+                or img.get("image")
+                or fullres
+            ).strip()
+            seen.add(fullres)
+            images.append(
+                ImageRecord(
+                    fullres_image=fullres,
+                    preview_image=preview or fullres,
+                    order_index=idx,
+                )
+            )
+        return images
+
+    @staticmethod
+    def _merge_image_records(existing: list[ImageRecord], fetched: list[ImageRecord]) -> list[ImageRecord]:
+        merged: list[ImageRecord] = []
+        seen: set[str] = set()
+
+        for source in (fetched, existing):
+            for image in source:
+                key = image.fullres_image.strip()
+                if not key or key in seen:
+                    continue
+                seen.add(key)
+                merged.append(image)
+
+        for idx, image in enumerate(merged):
+            image.order_index = idx
+
+        return merged
+
+    def _fetch_vehicle_images_with_retry(
+        self,
+        client: OpenLaneClient,
+        vehicle_id: str,
+    ) -> list[dict[str, Any]]:
+        retry_schedule = self.openlane_config.retry_schedule_seconds
+        for attempt in range(len(retry_schedule) + 1):
+            try:
+                return client.fetch_vehicle_images(vehicle_id)
+            except OpenLaneRequestError as exc:
+                if exc.status_code in {401, 403}:
+                    raise
+                is_retryable = exc.status_code == 429 or (
+                    exc.status_code is not None and exc.status_code >= 500
+                )
+                if not is_retryable or attempt >= len(retry_schedule):
+                    raise
+                delay = retry_schedule[attempt]
+                if exc.status_code == 429:
+                    delay = max(delay * 2, 8.0)
+                logger.warning(
+                    "Retry vehicle images vehicle_id=%s status=%s delay=%.1fs attempt=%d/%d",
+                    vehicle_id,
+                    exc.status_code,
+                    delay,
+                    attempt + 1,
+                    len(retry_schedule),
+                )
+                time.sleep(delay)
+
+        return []
+
+    def _enrich_car_records_with_images(
+        self,
+        authenticated_page,
+        raw_records: list[dict[str, Any]],
+        car_records: list[CarRecord],
+    ) -> int:
+        if not car_records:
+            return 0
+
+        raw_by_origin: dict[str, dict[str, Any]] = {}
+        for raw in raw_records:
+            raw_id = (
+                raw.get("id")
+                or raw.get("vehicle_id")
+                or raw.get("listing_id")
+                or raw.get("vin")
+            )
+            origin_id = self._normalize_origin_id(raw_id)
+            if origin_id:
+                raw_by_origin[origin_id] = raw
+
+        client = OpenLaneClient(authenticated_page, self.openlane_config)
+        images_added = 0
+
+        cars_by_vehicle_id: dict[str, CarRecord] = {}
+        for car in car_records:
+            raw = raw_by_origin.get(car.origin_id, {})
+            vehicle_id = raw.get("id") or raw.get("vehicle_id") or raw.get("listing_id")
+            if not vehicle_id:
+                # fallback: пробуем id из origin_id.
+                vehicle_id = car.origin_id.split(":", 1)[1] if ":" in car.origin_id else None
+            if not vehicle_id:
+                continue
+            cars_by_vehicle_id[str(vehicle_id)] = car
+
+        if not cars_by_vehicle_id:
+            return 0
+
+        vehicle_ids = list(cars_by_vehicle_id.keys())
+        try:
+            images_map = self._fetch_vehicle_images_batch_with_retry(client, vehicle_ids)
+        except OpenLaneRequestError as exc:
+            if exc.status_code in {401, 403}:
+                raise
+            logger.warning("Vehicle images batch fetch failed: %s", exc)
+            return 0
+        except Exception as exc:
+            logger.warning("Vehicle images batch fetch failed: %s", exc)
+            return 0
+
+        for vehicle_id, car in cars_by_vehicle_id.items():
+            images_payload = images_map.get(vehicle_id, [])
+            fetched_records = self._build_image_records(images_payload)
+            if not fetched_records:
+                continue
+
+            before_count = len(car.images)
+            # Source of truth: /api/vehicles/{id}/images.
+            car.images = fetched_records
+            after_count = len(car.images)
+            if after_count > before_count:
+                images_added += after_count - before_count
+
+        return images_added
+
+    def _fetch_vehicle_images_batch_with_retry(
+        self,
+        client: OpenLaneClient,
+        vehicle_ids: list[str],
+    ) -> dict[str, list[dict[str, Any]]]:
+        retry_schedule = self.openlane_config.retry_schedule_seconds
+        for attempt in range(len(retry_schedule) + 1):
+            try:
+                return client.fetch_vehicle_images_batch(vehicle_ids)
+            except OpenLaneRequestError as exc:
+                if exc.status_code in {401, 403}:
+                    raise
+                is_retryable = exc.status_code == 429 or (
+                    exc.status_code is not None and exc.status_code >= 500
+                )
+                if not is_retryable or attempt >= len(retry_schedule):
+                    raise
+                delay = retry_schedule[attempt]
+                if exc.status_code == 429:
+                    delay = max(delay * 2, 8.0)
+                logger.warning(
+                    "Retry vehicle images batch size=%d status=%s delay=%.1fs attempt=%d/%d",
+                    len(vehicle_ids),
+                    exc.status_code,
+                    delay,
+                    attempt + 1,
+                    len(retry_schedule),
+                )
+                time.sleep(delay)
+
+        return {}
+
+    def init_db(self) -> dict[str, Any]:
+        self.persistence.create_tables()
+        return {"status": "ok", "message": "DB tables created"}
+
+    def sync_listing(
+        self,
+        *,
+        lane: str | None = None,
+        limit: int | None = None,
+        only_new: bool | None = None,
+        max_pages: int | None = None,
+        concurrency: int | None = None,
+    ) -> dict[str, Any]:
+        """Полный цикл синхронизации: OpenLane API → фильтры → маппинг → DB.
+
+        Параметры берутся из аргументов → runtime_config.json → config.py (в этом порядке).
+        """
+        started_at = time.perf_counter()
+        rc = self.runtime_config
+
+        # Параметры: аргумент → runtime_config → config default.
+        lane = lane or rc.sync.lane or "openlane_marketplace"
+        if limit is None:
+            limit = rc.sync.limit
+        if limit is None and self.openlane_config.max_cars_limit > 0:
+            limit = self.openlane_config.max_cars_limit
+        if only_new is None:
+            only_new = rc.sync.only_new
+        effective_only_new = only_new if only_new is not None else False
+        max_pages = max_pages or self.openlane_config.max_pages
+        concurrency = max(1, min(concurrency or self.openlane_config.concurrency, 5))
+
+        self.persistence.create_tables()
+        run_id = self.persistence.start_sync_run(lane)
+
+        total_upserted = 0
+        total_failed = 0
+        total_images = 0
+        total_discovered = 0
+        total_skipped = 0
+        all_origin_ids: set[str] = set()
+        completed_pages: list[int] = []
+        failed_pages: list[int] = []
+        status = "success"
+        error_summary: str | None = None
+
+        # Предзагружаем известные origin_id для раннего пропуска.
+        known_ids: set[str] = set()
+        if effective_only_new:
+            known_ids = self.persistence.get_all_origin_ids_for_lane(prefix="openlane:")
+
+        logger.info(
+            "sync_listing started: lane=%s max_pages=%s concurrency=%s only_new=%s known=%d",
+            lane, max_pages, concurrency, effective_only_new, len(known_ids),
+        )
+
+        try:
+            with sync_playwright() as playwright:
+                browser = self.browser_factory.create_browser(playwright)
+                try:
+                    contexts = []
+                    auth_pages = []
+                    for i in range(concurrency):
+                        storage_state_path = (
+                            self.openlane_config.storage_state_file
+                            if Path(self.openlane_config.storage_state_file).exists()
+                            else None
+                        )
+                        ctx = self.browser_factory.create_context(browser, storage_state_path=storage_state_path)
+                        auth_page = self.authenticator.bootstrap_authenticated_context(ctx)
+                        contexts.append(ctx)
+                        auth_pages.append(auth_page)
+                        logger.info("Worker session initialized: worker=%d", i + 1)
+
+                    # Последовательная обработка страниц (по одной на контекст).
+                    # Для каждой страницы: fetch → map → upsert.
+                    context_idx = 0
+                    consecutive_all_known = 0
+                    consecutive_failures = 0
+                    early_stop_threshold = 3  # Остановка если 3 страницы подряд без новых записей.
+                    failure_circuit_breaker = 2  # Остановка если 2 подряд ошибки API.
+
+                    for page_num in range(1, max_pages + 1):
+                        if limit is not None and total_upserted >= limit:
+                            logger.info("Reached limit=%d, stopping", limit)
+                            break
+
+                        if consecutive_failures >= failure_circuit_breaker:
+                            logger.error(
+                                "Circuit breaker: %d consecutive failures — stopping to protect account",
+                                consecutive_failures,
+                            )
+                            status = "aborted"
+                            break
+
+                        ctx = auth_pages[context_idx % len(auth_pages)]
+                        context_idx += 1
+
+                        try:
+                            page_result = self._fetch_page_with_retry(ctx, page_num)
+                            consecutive_failures = 0  # Сброс при успехе.
+                        except Exception as exc:
+                            logger.error("Page %d fetch failed: %s", page_num, exc)
+                            failed_pages.append(page_num)
+                            total_failed += 1
+                            consecutive_failures += 1
+                            continue
+
+                        if not page_result.records:
+                            logger.info("Page %d returned 0 records — end of listing", page_num)
+                            completed_pages.append(page_num)
+                            break
+
+                        # Применяем runtime-фильтры к сырым записям.
+                        filtered_records = apply_filters(page_result.records, rc.filters)
+                        if len(filtered_records) < len(page_result.records):
+                            logger.debug(
+                                "Page %d: %d/%d records passed runtime filters",
+                                page_num, len(filtered_records), len(page_result.records),
+                            )
+
+                        # Маппинг JSON → CarRecord.
+                        car_records = map_openlane_records(filtered_records)
+                        total_discovered += len(page_result.records)
+
+                        if not car_records:
+                            logger.warning("Page %d: all %d records failed mapping", page_num, len(page_result.records))
+                            completed_pages.append(page_num)
+                            continue
+
+                        # Фильтрация уже известных записей.
+                        if effective_only_new and known_ids:
+                            new_records = [r for r in car_records if r.origin_id not in known_ids]
+                            skipped = len(car_records) - len(new_records)
+                            total_skipped += skipped
+                            if skipped:
+                                logger.debug("Page %d: skipped %d already known", page_num, skipped)
+                            if not new_records:
+                                consecutive_all_known += 1
+                                completed_pages.append(page_num)
+                                if consecutive_all_known >= early_stop_threshold:
+                                    logger.info(
+                                        "Early stop: %d consecutive pages with all known records",
+                                        consecutive_all_known,
+                                    )
+                                    break
+                                continue
+                            else:
+                                consecutive_all_known = 0
+                            car_records = new_records
+
+                        # Применяем limit.
+                        if limit is not None:
+                            remaining = limit - total_upserted
+                            car_records = car_records[:remaining]
+
+                        # Дозапрашиваем фото для машин, у которых их нет в search payload.
+                        try:
+                            added_images = self._enrich_car_records_with_images(ctx, filtered_records, car_records)
+                            if added_images:
+                                logger.debug("Page %d: enriched %d images via vehicle images API", page_num, added_images)
+                        except OpenLaneRequestError as exc:
+                            if exc.status_code in {401, 403}:
+                                raise
+                            logger.warning("Page %d image enrichment skipped: %s", page_num, exc)
+
+                        # Upsert батчем.
+                        try:
+                            upsert_result = self.persistence.upsert_cars_batch(car_records)
+                            page_upserted = upsert_result.get("inserted", 0) + upsert_result.get("updated", 0)
+                            page_images = upsert_result.get("images_upserted", 0)
+                            total_upserted += page_upserted
+                            total_images += page_images
+                            for r in car_records:
+                                all_origin_ids.add(r.origin_id)
+                                known_ids.add(r.origin_id)
+                            logger.info(
+                                "Page %d: %d records → %d upserted, %d images",
+                                page_num, len(car_records), page_upserted, page_images,
+                            )
+                        except Exception as exc:
+                            logger.error("Page %d upsert failed: %s", page_num, exc, exc_info=True)
+                            total_failed += len(car_records)
+                            failed_pages.append(page_num)
+                            continue
+
+                        completed_pages.append(page_num)
+
+                finally:
+                    browser.close()
+
+        except Exception as exc:
+            status = "failed"
+            error_summary = str(exc)[:500]
+            logger.error("sync_listing failed: %s", exc, exc_info=True)
+
+        elapsed = time.perf_counter() - started_at
+        self.persistence.finish_sync_run(
+            run_id,
+            status=status,
+            ids_fetched=total_discovered,
+            cars_upserted=total_upserted,
+            cars_failed=total_failed,
+            images_upserted=total_images,
+            error_summary=error_summary,
+        )
+
+        result = {
+            "run_id": run_id,
+            "status": status,
+            "lane": lane,
+            "total_discovered": total_discovered,
+            "cars_upserted": total_upserted,
+            "cars_failed": total_failed,
+            "images_upserted": total_images,
+            "skipped_existing": total_skipped,
+            "completed_pages": len(completed_pages),
+            "failed_pages": len(failed_pages),
+            "elapsed_seconds": round(elapsed, 2),
+            "full_scan_completed": status == "success",
+        }
+        logger.info("sync_listing finished: %s", result)
+        return result
+
+    def _fetch_page_with_retry(self, authenticated_page, page: int) -> OpenLanePageResult:
+        client = OpenLaneClient(authenticated_page, self.openlane_config)
+        retry_schedule = self.openlane_config.retry_schedule_seconds
+
+        for attempt in range(len(retry_schedule) + 1):
+            try:
+                return client.fetch_page(page)
+            except OpenLaneRequestError as exc:
+                # 403 — возможная блокировка, стоп.
+                if exc.status_code == 403:
+                    logger.error(
+                        "STOP: page=%d returned 403 Forbidden — aborting to protect account",
+                        page,
+                    )
+                    raise
+                # 401 — сессия истекла.
+                if exc.status_code == 401:
+                    raise
+                # 429 — rate limit, ретрай с увеличенным backoff.
+                is_retryable = exc.status_code == 429 or (
+                    exc.status_code is not None and exc.status_code >= 500
+                )
+                if not is_retryable or attempt >= len(retry_schedule):
+                    raise
+                delay = retry_schedule[attempt]
+                if exc.status_code == 429:
+                    delay = max(delay * 3, 15.0)  # 429 → утроенная задержка
+                logger.warning(
+                    "Retry page=%d status=%s delay=%.1fs attempt=%d/%d",
+                    page, exc.status_code, delay, attempt + 1, len(retry_schedule),
+                )
+                time.sleep(delay)
+            except Exception:
+                # Неожиданная ошибка — одна попытка.
+                if attempt >= min(1, len(retry_schedule)):
+                    raise
+                delay = retry_schedule[attempt] if attempt < len(retry_schedule) else 5.0
+                logger.warning(
+                    "Transient error page=%d delay=%.1fs attempt=%d",
+                    page, delay, attempt + 1,
+                    exc_info=True,
+                )
+                time.sleep(delay)
+
+        raise RuntimeError(f"Page {page} exhausted retries")

openlane_scraper/storage/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

openlane_scraper/storage/db.py

@@ -0,0 +1,576 @@
+import logging
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from typing import Any, Iterator
+
+from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.orm import Session, sessionmaker
+
+from ..core.config import Settings
+from .models import Base, Car, Image, SyncRun
+from .schemas import CarRecord
+
+logger = logging.getLogger("openlane_scraper.db")
+
+
+CAR_DB_FIELDS = {
+	col.key for col in Car.__table__.columns
+	if col.key not in ("id",)
+}
+
+_IN_CHUNK_SIZE = 5000
+
+
+class PersistenceService:
+
+	def __init__(self, settings: Settings) -> None:
+		self.settings = settings
+		engine_kwargs = {
+			"echo": settings.database.echo,
+			"future": True,
+		}
+		if "postgresql" in settings.database.url:
+			engine_kwargs["pool_size"] = settings.database.pool_size
+			engine_kwargs["max_overflow"] = settings.database.max_overflow
+			engine_kwargs["pool_pre_ping"] = True
+			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
+		self.engine = create_engine(settings.database.url, **engine_kwargs)
+		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
+
+	def create_tables(self) -> None:
+		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
+		is_sqlite = self.settings.database.url.startswith("sqlite")
+		if not is_sqlite and not self.settings.database.auto_create_tables:
+			return
+		try:
+			Base.metadata.create_all(self.engine)
+		except Exception:
+			logger.debug("create_tables skipped (schema already exists)")
+
+	@contextmanager
+	def session_scope(self) -> Iterator[Session]:
+		session = self.session_factory()
+		try:
+			yield session
+			session.commit()
+		except Exception:
+			session.rollback()
+			raise
+		finally:
+			session.close()
+
+	def start_sync_run(self, lane: str) -> int:
+		with self.session_scope() as session:
+			now = datetime.now(timezone.utc)
+			stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
+			for stale in stale_runs:
+				stale.status = "failed"
+				stale.finished_at = now
+				if not stale.error_summary:
+					stale.error_summary = "Recovered stale running sync run before starting a new run"
+
+			run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
+			session.add(run)
+			session.flush()
+			return int(run.id)
+
+	def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
+		with self.session_scope() as session:
+			run = session.get(SyncRun, run_id)
+			if run is None:
+				return
+			run.finished_at = datetime.now(timezone.utc)
+			run.status = status
+			run.ids_fetched = ids_fetched
+			run.cars_upserted = cars_upserted
+			run.cars_failed = cars_failed
+			run.images_upserted = images_upserted
+			run.error_summary = error_summary
+
+	def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
+		if not origin_urls:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
+		if not origin_ids:
+			return set()
+		with self.session_scope() as session:
+			rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
+			return {str(row[0]) for row in rows if row and row[0]}
+
+	def get_existing_urls_and_ids(
+		self, origin_urls: list[str], origin_ids: list[str],
+	) -> tuple[set[str], set[str]]:
+		# Загрузка существующих URL и origin_id.
+		if not origin_urls and not origin_ids:
+			return set(), set()
+		urls: set[str] = set()
+		ids: set[str] = set()
+		with self.session_scope() as session:
+			max_len = max(len(origin_urls), len(origin_ids), 1)
+			for i in range(0, max_len, _IN_CHUNK_SIZE):
+				url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+				id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+				conditions = []
+				if url_chunk:
+					conditions.append(Car.origin_url.in_(url_chunk))
+				if id_chunk:
+					conditions.append(Car.origin_id.in_(id_chunk))
+				if not conditions:
+					continue
+				rows = session.execute(
+					select(Car.origin_url, Car.origin_id).where(or_(*conditions))
+				).all()
+				for r in rows:
+					if r[0]:
+						urls.add(str(r[0]))
+					if r[1]:
+						ids.add(str(r[1]))
+		return urls, ids
+
+	@staticmethod
+	def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
+		if not images:
+			return
+		session.add_all([
+			Image(
+				fullres_image=str(img["fullres_image"]),
+				preview_image=str(img["preview_image"]),
+				order_index=int(img.get("order_index", 0)),
+				car_id=car_id,
+			)
+			for img in images
+		])
+
+	@staticmethod
+	def _car_payload(record: CarRecord) -> dict[str, object]:
+		payload = record.model_dump(mode="python")
+		return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
+
+	def _is_postgres(self) -> bool:
+		return self.engine.dialect.name == "postgresql"
+
+	def _load_existing_cars(
+		self,
+		session: Session,
+		origin_ids: list[str],
+		origin_urls: list[str],
+	) -> tuple[dict[str, Car], dict[str, Car]]:
+		existing_by_id: dict[str, Car] = {}
+		existing_by_url: dict[str, Car] = {}
+		if not origin_ids and not origin_urls:
+			return existing_by_id, existing_by_url
+
+		existing_cars: list[Car] = []
+		max_len = max(len(origin_ids), len(origin_urls), 1)
+		for i in range(0, max_len, _IN_CHUNK_SIZE):
+			id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
+			url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
+			conditions = []
+			if id_chunk:
+				conditions.append(Car.origin_id.in_(id_chunk))
+			if url_chunk:
+				conditions.append(Car.origin_url.in_(url_chunk))
+			if not conditions:
+				continue
+			rows = session.execute(
+				select(Car).where(or_(*conditions))
+			).scalars().all()
+			existing_cars.extend(rows)
+
+		for car in existing_cars:
+			if car.origin_id:
+				existing_by_id[car.origin_id] = car
+			if car.origin_url:
+				existing_by_url[car.origin_url] = car
+		return existing_by_id, existing_by_url
+
+	def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
+		existing_images_map: dict[int, set[str]] = {}
+		if not car_ids:
+			return existing_images_map
+
+		car_id_list = list(car_ids)
+		for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
+			chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
+			img_rows = session.execute(
+				select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
+			).all()
+			for cid, furl in img_rows:
+				existing_images_map.setdefault(int(cid), set()).add(str(furl))
+		return existing_images_map
+
+	@staticmethod
+	def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
+		return {
+			key: getattr(insert_stmt.excluded, key)
+			for key in CAR_DB_FIELDS
+		}
+
+	def _replace_images_for_car(
+		self,
+		session: Session,
+		car_id: int,
+		images: list[dict[str, object]],
+		origin_id: str,
+	) -> int:
+		nested = session.begin_nested()
+		try:
+			session.execute(delete(Image).where(Image.car_id == car_id))
+			self._add_images(session, car_id, images)
+			session.flush()
+			nested.commit()
+			return len(images)
+		except Exception:
+			nested.rollback()
+			logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
+			return 0
+
+	def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			entries: list[dict[str, object]] = []
+			upsert_payloads: list[dict[str, object]] = []
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+				car_by_id = existing_by_id.get(record.origin_id)
+				car_by_url = existing_by_url.get(record.origin_url)
+				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					entry["car_id"] = int(car_by_url.id)
+					updated += 1
+				else:
+					upsert_payloads.append(payload)
+					if car_by_id is not None:
+						updated += 1
+					else:
+						inserted += 1
+				entries.append(entry)
+
+			session.flush()
+
+			if upsert_payloads:
+				insert_stmt = pg_insert(Car).values(upsert_payloads)
+				upsert_stmt = insert_stmt.on_conflict_do_update(
+					index_elements=[Car.origin_id],
+					set_=self._postgres_upsert_set_map(insert_stmt),
+				).returning(Car.id, Car.origin_id)
+				rows = session.execute(upsert_stmt).all()
+				car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
+				for entry in entries:
+					if entry["car_id"] is not None:
+						continue
+					record = entry["record"]
+					car_id = car_ids_by_origin_id.get(record.origin_id)
+					if car_id is None:
+						raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
+					entry["car_id"] = car_id
+
+			car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
+			existing_images_map = self._load_existing_image_urls(session, car_ids)
+			images_by_car_id: dict[int, list[dict[str, object]]] = {}
+			replace_ids: list[int] = []
+
+			for entry in entries:
+				car_id = int(entry["car_id"])
+				images = entry["images"]
+				new_image_urls = {
+					str(img.get("fullres_image", ""))
+					for img in images
+					if img.get("fullres_image")
+				}
+				old_image_urls = existing_images_map.get(car_id, set())
+				if new_image_urls != old_image_urls:
+					replace_ids.append(car_id)
+					images_by_car_id[car_id] = images
+				else:
+					images_total += len(old_image_urls)
+
+			if replace_ids:
+				for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
+					chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car_id in replace_ids:
+					images = images_by_car_id[car_id]
+					self._add_images(session, car_id, images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def upsert_car(self, record: CarRecord):
+		# Вставка или обновление автомобиля по origin_id/origin_url.
+		payload = self._car_payload(record)
+		images = [image.model_dump(mode="python") for image in record.images]
+		with self.session_scope() as session:
+			if self._is_postgres():
+				car_by_url = session.execute(
+					select(Car).where(Car.origin_url == record.origin_url)
+				).scalar_one_or_none()
+				if car_by_url is not None and car_by_url.origin_id != record.origin_id:
+					for key, value in payload.items():
+						setattr(car_by_url, key, value)
+					car_by_url.last_seen_at = record.last_seen_at
+					session.flush()
+					car_id = int(car_by_url.id)
+					action = "updated"
+				else:
+					existed = session.execute(
+						select(Car.id).where(Car.origin_id == record.origin_id)
+					).scalar_one_or_none() is not None
+					insert_stmt = pg_insert(Car).values(**payload)
+					upsert_stmt = insert_stmt.on_conflict_do_update(
+						index_elements=[Car.origin_id],
+						set_=self._postgres_upsert_set_map(insert_stmt),
+					).returning(Car.id)
+					car_id = int(session.execute(upsert_stmt).scalar_one())
+					action = "updated" if existed or car_by_url is not None else "inserted"
+
+				images_upserted = self._replace_images_for_car(
+					session, car_id, images, record.origin_id,
+				)
+				return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
+
+			car = session.execute(
+				select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
+			).scalar_one_or_none()
+			action = "inserted"
+			if car is None:
+				car = Car(**payload)
+				session.add(car)
+				session.flush()
+			else:
+				action = "updated"
+				for key, value in payload.items():
+					setattr(car, key, value)
+				car.last_seen_at = record.last_seen_at
+				session.flush()
+				images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
+				return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
+			self._add_images(session, int(car.id), images)
+			session.flush()
+			return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
+	def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Пакетный upsert нескольких автомобилей в одной транзакции.
+
+		Оптимизации:
+		- Дедупликация записей по origin_id перед вставкой.
+		- Chunked IN-queries для больших списков (обход лимита PG параметров).
+		- Пропуск перезаписи изображений, если набор URL не изменился.
+		- Один DELETE по car_id IN (...) вместо удаления по одному.
+		- Fallback на по-одному upsert если batch commit упал.
+		"""
+		# ── Дедупликация записей внутри батча ──
+		seen_ids: dict[str, int] = {}
+		unique_records: list[CarRecord] = []
+		for idx, r in enumerate(records):
+			key = r.origin_id or r.origin_url
+			if key in seen_ids:
+				logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
+				continue
+			seen_ids[key] = idx
+			unique_records.append(r)
+
+		if len(unique_records) < len(records):
+			logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
+		records = unique_records
+
+		try:
+			return self._upsert_cars_batch_inner(records)
+		except Exception as exc:
+			logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
+			return self._upsert_cars_individually(records)
+
+	def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
+		"""Внутренняя реализация batched upsert (одна транзакция)."""
+		if self._is_postgres():
+			return self._upsert_cars_batch_postgres(records)
+
+		inserted = 0
+		updated = 0
+		images_total = 0
+
+		with self.session_scope() as session:
+			# Получаем существующие записи chunked-запросами.
+			origin_ids = [r.origin_id for r in records if r.origin_id]
+			origin_urls = [r.origin_url for r in records if r.origin_url]
+
+			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
+
+			# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
+			existing_car_ids = set()
+			for record in records:
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is not None:
+					existing_car_ids.add(int(car.id))
+
+			# Строим маппинг car_id → set(image_urls) для сравнения.
+			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
+
+			new_cars: list[tuple[Car, list[dict]]] = []
+			update_cars_needing_images: list[tuple[Car, list[dict]]] = []
+
+			for record in records:
+				payload = self._car_payload(record)
+				images = [image.model_dump(mode="python") for image in record.images]
+
+				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
+				if car is None:
+					car = Car(**payload)
+					session.add(car)
+					inserted += 1
+					new_cars.append((car, images))
+				else:
+					for key, value in payload.items():
+						setattr(car, key, value)
+					car.last_seen_at = record.last_seen_at
+					updated += 1
+
+					# Проверяем, изменились ли изображения.
+					new_image_urls = {img.get("fullres_image", "") for img in images}
+					old_image_urls = existing_images_map.get(int(car.id), set())
+					if new_image_urls != old_image_urls:
+						update_cars_needing_images.append((car, images))
+					else:
+						images_total += len(old_image_urls)
+
+			# Один flush для всех вставок.
+			session.flush()
+
+			# Добавляем изображения для новых автомобилей.
+			for car, images in new_cars:
+				self._add_images(session, int(car.id), images)
+				images_total += len(images)
+
+			# Массово обновляем изображения только для машин с изменёнными картинками.
+			if update_cars_needing_images:
+				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
+				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
+					chunk = update_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(delete(Image).where(Image.car_id.in_(chunk)))
+				for car, images in update_cars_needing_images:
+					self._add_images(session, int(car.id), images)
+					images_total += len(images)
+
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
+		# Fallback на поштучный upsert.
+		inserted = 0
+		updated = 0
+		images_total = 0
+		for record in records:
+			try:
+				result = self.upsert_car(record)
+				action = result.get("action", "inserted")
+				if action == "inserted":
+					inserted += 1
+				else:
+					updated += 1
+				images_total += int(result.get("images_upserted", 0))
+			except Exception as exc:
+				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
+		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
+
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "openlane") -> int:
+		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
+		if not active_origin_ids:
+			return 0
+		with self.session_scope() as session:
+			stmt = (
+				update(Car)
+				.where(Car.origin_id.notin_(active_origin_ids))
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like("openlane:%"))
+				.values(is_sold=True)
+			)
+			result = session.execute(stmt)
+			count = result.rowcount or 0
+			if count:
+				logger.info("Marked %d cars as sold (no longer in listing)", count)
+			return count
+
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "openlane") -> int:
+		"""Помечает авто как проданные, если их URL нет в активном листинге.
+
+		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
+		что кардинально быстрее при больших объёмах (100K+ URLs).
+		"""
+		if not active_origin_urls:
+			return 0
+
+		is_postgres = "postgresql" in self.settings.database.url
+
+		with self.session_scope() as session:
+			if is_postgres:
+				# Создаём временную таблицу с активными URL.
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
+				session.execute(text("TRUNCATE _active_urls"))
+
+				# Вставляем активные URL чанками.
+				url_list = list(active_origin_urls)
+				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
+					chunk = url_list[i:i + _IN_CHUNK_SIZE]
+					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
+					params = {f"u{j}": url for j, url in enumerate(chunk)}
+					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
+
+				# Создаём индекс на временной таблице для ускорения JOIN.
+				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
+
+				# Массовая пометка проданных в PostgreSQL.
+				result = session.execute(text("""
+					UPDATE cars
+					SET is_sold = TRUE
+					FROM (
+						SELECT c.id
+						FROM cars c
+						LEFT JOIN _active_urls a ON c.origin_url = a.url
+						WHERE a.url IS NULL
+						AND c.is_sold = FALSE
+						AND c.origin_id LIKE 'openlane:%%'
+					) sub
+					WHERE cars.id = sub.id
+				"""))
+				count = result.rowcount or 0
+			else:
+				# Упрощённый путь для SQLite.
+				stmt = (
+					update(Car)
+					.where(Car.origin_url.notin_(active_origin_urls))
+					.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like("openlane:%"))
+				.values(is_sold=True)
+				)
+				result = session.execute(stmt)
+				count = result.rowcount or 0
+
+			if count:
+				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
+			return count
+
+	def get_all_origin_ids_for_lane(self, prefix: str = "openlane:") -> set[str]:
+		"""Возвращает все известные origin_id для заданного префикса.
+
+		Использует yield_per для потоковой загрузки при большом количестве записей.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}

openlane_scraper/storage/enums.py

@@ -0,0 +1,25 @@
+CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
+DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA")
+GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
+STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA")
+BODY_TYPE_ENUM_VALUES = (
+	"COUPE",
+	"SUV",
+	"HATCHBACK",
+	"MINIVAN",
+	"SEDAN",
+	"STATION_WAGON",
+	"PICKUP",
+	"TRUCK",
+	"OPEN",
+	"RV",
+	"OTHER",
+	"NA",
+)
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
+ORIGIN_ENUM_VALUES = (
+	"OPENLANE",
+	"IAAI",
+	"NA",
+)
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

openlane_scraper/storage/models.py

@@ -0,0 +1,82 @@
+from datetime import datetime
+
+from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+
+from .enums import (
+	BODY_TYPE_ENUM_VALUES,
+	COUNTRY_ENUM_VALUES,
+	CURRENCY_ENUM_VALUES,
+	DRIVE_ENUM_VALUES,
+	GEARBOX_ENUM_VALUES,
+	ORIGIN_ENUM_VALUES,
+	SELLING_TYPE_ENUM_VALUES,
+	STEERING_WHEEL_ENUM_VALUES,
+)
+
+
+class Base(DeclarativeBase):
+	pass
+
+
+class Car(Base):
+	__tablename__ = "cars"
+	__table_args__ = (
+		Index("ix_cars_brand_model", "brand", "model"),
+		Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
+	brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
+	model: Mapped[str] = mapped_column(String(50), nullable=False)
+	year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
+	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
+	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
+	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
+	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
+	gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True)
+	steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True)
+	body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER")
+	engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
+	selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
+	origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
+	is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
+	non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+	rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+	slug: Mapped[str] = mapped_column(String(), nullable=False)
+	last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
+
+
+class Image(Base):
+	__tablename__ = "images"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
+	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
+	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
+	car: Mapped[Car] = relationship("Car", back_populates="images")
+
+
+class SyncRun(Base):
+	__tablename__ = "sync_runs"
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	status: Mapped[str] = mapped_column(Text, nullable=False, index=True)
+	lane: Mapped[str] = mapped_column(Text, nullable=False)
+	ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)

openlane_scraper/storage/schemas.py

@@ -0,0 +1,87 @@
+from datetime import datetime, timezone
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ImageRecord(BaseModel):
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0
+
+
+class CarRecord(BaseModel):
+	parser_id: str
+	brand: str
+	model: str
+	year: int | None = None
+	price: int | None = None
+	currency: str = "USD"
+	mileage: int = 0
+	country: str = "US"
+	is_sold: bool = False
+	color: str = "other"
+	drive: str | None = None
+	gearbox: str | None = None
+	steering_wheel: str | None = None
+	body_type: str = "OTHER"
+	engine_volume: int | None = None
+	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
+	origin: str = "NA"
+	origin_url: str
+	origin_id: str
+	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
+	slug: str
+	last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+	images: list[ImageRecord] = Field(default_factory=list)
+
+
+class ImageRead(BaseModel):
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	fullres_image: str
+	preview_image: str
+	order_index: int = 0
+
+
+class CarRead(BaseModel):
+	model_config = ConfigDict(from_attributes=True)
+
+	id: int
+	parser_id: str
+	brand: str
+	model: str
+	year: int | None = None
+	price: int | None = None
+	currency: str = "USD"
+	mileage: int = 0
+	country: str = "US"
+	is_sold: bool = False
+	color: str = "other"
+	drive: str | None = None
+	gearbox: str | None = None
+	steering_wheel: str | None = None
+	body_type: str = "OTHER"
+	engine_volume: int | None = None
+	selling_type: str = "AUCTION"
+	one_owner: bool = False
+	new_car: bool = False
+	is_hidden: bool = False
+	origin: str = "NA"
+	origin_url: str = ""
+	origin_id: str = ""
+	is_damaged: bool = False
+	evaluation: str | None = None
+	non_smoking: bool = True
+	rental: bool = False
+	repair_history: bool = False
+	slug: str = ""
+	last_seen_at: datetime | None = None
+	images: list[ImageRead] = Field(default_factory=list)
+

openlane_scraper/worker/__init__.py

@@ -0,0 +1,3 @@
+from .celery_app import celery_app
+
+__all__ = ["celery_app"]

openlane_scraper/worker/celery_app.py

@@ -0,0 +1,124 @@
+# Инициализация Celery-приложения и периодических задач.
+
+import logging
+
+from celery import Celery
+from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
+from redis import Redis
+
+from ..core.config import settings
+from ..core.logs import setup_logging
+
+logger = logging.getLogger("openlane_scraper.worker.celery_app")
+STARTUP_SYNC_DISPATCH_KEY = "openlane:state:startup_sync_dispatched"
+
+
+@celery_setup_logging.connect
+def _configure_logging(loglevel=None, **kwargs):
+    # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+@worker_process_init.connect
+def _on_worker_process_init(**kwargs):
+    # Повторно настраиваем логирование в каждом дочернем prefork-процессе,
+    # чтобы StreamHandler(stderr) корректно работал после fork.
+    level = settings.log_level if settings.log_level else "INFO"
+    setup_logging(level, None)
+
+
+def _broker_url() -> str:
+    return settings.celery.broker_url or settings.redis.url
+
+
+def _result_backend() -> str:
+    return settings.celery.result_backend or settings.redis.url
+
+
+celery_app = Celery(
+    "openlane_scraper",
+    broker=_broker_url(),
+    backend=_result_backend(),
+)
+
+# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
+# ограничиваем, чтобы зависший worker не жил вечно.
+_soft = settings.celery.task_soft_time_limit
+_hard = settings.celery.task_time_limit
+_max_hard = _soft + 120 if _soft else _hard
+if _hard > _max_hard:
+    logger.warning(
+        "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
+        "clamping hard limit to %d",
+        _hard, _soft, _max_hard,
+    )
+    _hard = _max_hard
+
+celery_app.conf.update(
+    task_serializer="json",
+    accept_content=["json"],
+    result_serializer="json",
+    timezone="UTC",
+    enable_utc=True,
+    task_soft_time_limit=_soft,
+    task_time_limit=_hard,
+    task_acks_late=True,
+    task_reject_on_worker_lost=True,
+    task_track_started=True,
+    worker_concurrency=settings.celery.worker_concurrency,
+    worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
+    worker_pool="prefork",
+    worker_prefetch_multiplier=1,
+    broker_connection_retry_on_startup=True,
+    broker_transport_options={
+        "visibility_timeout": settings.celery.broker_visibility_timeout,
+    },
+    result_expires=86400,
+    worker_redirect_stdouts=False,
+    worker_hijack_root_logger=False,
+    beat_schedule={
+        "periodic-sync-listing": {
+            "task": "openlane_scraper.worker.tasks.sync_listing_task",
+            "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
+            "args": (),
+            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False},
+            "options": {"queue": "scraping"},
+        }
+    },
+    task_routes={
+        "openlane_scraper.worker.tasks.*": {"queue": "scraping"}
+    },
+)
+
+celery_app.autodiscover_tasks(["openlane_scraper.worker"])
+
+
+@worker_ready.connect
+def _on_worker_ready(**kwargs):
+    """Сразу при старте worker отправляем первую задачу sync_listing,
+    чтобы не ждать час до первого beat-цикла."""
+    try:
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+    except Exception:
+        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        return
+
+    if not should_dispatch:
+        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
+        return
+
+    logger.info("Worker ready — dispatching initial sync_listing task")
+    celery_app.send_task(
+        "openlane_scraper.worker.tasks.sync_listing_task",
+        kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
+        queue="scraping",
+    )

openlane_scraper/worker/tasks.py

@@ -0,0 +1,360 @@
+# Задачи Celery для синхронизации OpenLane marketplace → DB.
+
+import logging
+import random
+import time
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from threading import Event, Thread
+
+from billiard.exceptions import SoftTimeLimitExceeded
+from celery import shared_task
+from redis import Redis
+
+from ..core.config import Settings
+from ..scraper import OpenLaneScraper
+from ..storage.db import PersistenceService
+
+logger = logging.getLogger("openlane_scraper.worker.tasks")
+
+SYNC_LISTING_LOCK_KEY = "openlane:locks:sync_listing"
+SYNC_FULL_SCAN_DONE_KEY = "openlane:state:sync_full_scan_done"
+SYNC_LISTING_TASK_NAME = "openlane_scraper.worker.tasks.sync_listing_task"
+
+
+def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0):
+    last_exc: Exception | None = None
+    for attempt in range(1, attempts + 1):
+        try:
+            return func()
+        except Exception as exc:
+            last_exc = exc
+            if attempt >= attempts:
+                break
+            delay = base_delay_s * (2 ** (attempt - 1))
+            logger.warning(
+                "Operation failed (attempt %d/%d): %s. Retrying in %.1fs",
+                attempt, attempts, exc, delay,
+            )
+            time.sleep(delay)
+    if last_exc is not None:
+        raise last_exc
+
+
+def _run_browser_job(func, *args, **kwargs):
+    settings = Settings()
+    soft = settings.celery.task_soft_time_limit
+    hard = settings.celery.task_time_limit
+    wait_timeout = min(hard, soft + 120) if soft and hard else None
+
+    executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="openlane-browser")
+    future = executor.submit(func, *args, **kwargs)
+    try:
+        result = future.result(timeout=wait_timeout)
+    except SoftTimeLimitExceeded:
+        future.cancel()
+        executor.shutdown(wait=False, cancel_futures=True)
+        raise
+    except TimeoutError:
+        future.cancel()
+        executor.shutdown(wait=False, cancel_futures=True)
+        raise SoftTimeLimitExceeded("Browser thread did not finish within time limit")
+    except Exception:
+        executor.shutdown(wait=False, cancel_futures=True)
+        raise
+    else:
+        executor.shutdown(wait=True)
+    return result
+
+
+def _sync_listing_lock_ttl_seconds() -> int:
+    settings = Settings()
+    soft = settings.celery.task_soft_time_limit
+    hard = settings.celery.task_time_limit
+    effective_hard = min(hard, soft + 120) if soft else hard
+    return max(effective_hard + 120, 300)
+
+
+def _get_persistence() -> PersistenceService:
+    settings = Settings()
+    persistence = PersistenceService(settings)
+
+    def _ping_db() -> None:
+        with persistence.engine.connect() as conn:
+            conn.exec_driver_sql("SELECT 1")
+
+    _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0)
+    return persistence
+
+
+def _get_redis() -> Redis:
+    settings = Settings()
+    redis_client = Redis.from_url(
+        settings.redis.url,
+        decode_responses=True,
+        socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+        socket_timeout=settings.redis.socket_timeout_seconds,
+        health_check_interval=settings.redis.health_check_interval_seconds,
+        retry_on_timeout=True,
+    )
+
+    def _ping_redis() -> None:
+        redis_client.ping()
+
+    _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0)
+    return redis_client
+
+
+def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
+    try:
+        acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
+        if acquired:
+            return True
+        ttl = redis_client.ttl(key)
+        if ttl is not None and ttl < 0:
+            logger.warning("Detected stale lock without TTL, removing: %s", key)
+            redis_client.delete(key)
+            return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
+        return False
+    except Exception:
+        logger.warning("Failed to acquire lock %s", key, exc_info=True)
+        return False
+
+
+def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None:
+    try:
+        refreshed = redis_client.eval(
+            """
+            if redis.call('GET', KEYS[1]) == ARGV[1] then
+                return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
+            end
+            return 0
+            """,
+            1, key, owner_token, int(ttl_seconds),
+        )
+        return bool(refreshed)
+    except Exception:
+        logger.warning("Failed to refresh lock %s", key, exc_info=True)
+        return None
+
+
+def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None:
+    try:
+        redis_client.eval(
+            """
+            if redis.call('GET', KEYS[1]) == ARGV[1] then
+                return redis.call('DEL', KEYS[1])
+            end
+            return 0
+            """,
+            1, key, owner_token,
+        )
+    except Exception:
+        logger.warning("Failed to release lock %s", key, exc_info=True)
+
+
+def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool:
+    try:
+        inspector = celery_app.control.inspect(timeout=1.0)
+        snapshots = [
+            inspector.active() or {},
+            inspector.reserved() or {},
+            inspector.scheduled() or {},
+        ]
+    except Exception:
+        logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True)
+        return True
+
+    for snapshot in snapshots:
+        for entries in snapshot.values():
+            for entry in entries or []:
+                task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
+                if task_name != SYNC_LISTING_TASK_NAME:
+                    continue
+                entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "")
+                if exclude_task_id and entry_id == exclude_task_id:
+                    continue
+                return True
+    return False
+
+
+def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool:
+    try:
+        owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY)
+        if not owner_token:
+            return False
+    except Exception:
+        logger.warning("Failed to read sync listing lock before cleanup", exc_info=True)
+        return False
+
+    if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id):
+        logger.info("sync_listing lock preserved: active task still detected")
+        return False
+
+    try:
+        ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY)
+        redis_client.delete(SYNC_LISTING_LOCK_KEY)
+        logger.warning(
+            "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart",
+            owner_token, ttl,
+        )
+        return True
+    except Exception:
+        logger.warning("Failed to clear orphan sync listing lock", exc_info=True)
+        return False
+
+
+def _is_full_scan_done(redis_client: Redis) -> bool:
+    try:
+        value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY)
+    except Exception:
+        logger.warning("Failed to read full scan state", exc_info=True)
+        return False
+    return str(value or "").strip() == "1"
+
+
+def _set_full_scan_done(redis_client: Redis, done: bool) -> None:
+    try:
+        redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0")
+    except Exception:
+        logger.warning("Failed to persist full scan state", exc_info=True)
+
+
+def _start_lock_heartbeat(
+    redis_client: Redis, key: str, owner_token: str, ttl_seconds: int,
+) -> tuple[Event, Thread]:
+    stop_event = Event()
+    interval_seconds = max(5.0, min(30.0, ttl_seconds / 3))
+
+    def _heartbeat() -> None:
+        while not stop_event.wait(interval_seconds):
+            refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds)
+            if refreshed is False:
+                logger.warning("Lost sync_listing lock ownership for %s", owner_token)
+                return
+
+    thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True)
+    thread.start()
+    return stop_event, thread
+
+
+@shared_task(
+    name="openlane_scraper.worker.tasks.sync_listing_task",
+    bind=True,
+    max_retries=3,
+    default_retry_delay=120,
+    acks_late=True,
+)
+def sync_listing_task(
+    self,
+    lane: str = "openlane_marketplace",
+    limit: int | None = None,
+    only_new: bool | None = None,
+    max_pages: int | None = None,
+    concurrency: int | None = None,
+):
+    """Полный цикл синхронизации OpenLane marketplace → DB."""
+    persistence = _get_persistence()
+    persistence.create_tables()
+    task_id = self.request.id or "unknown"
+    owner_token = f"{task_id}:{uuid.uuid4().hex}"
+    redis_client = _get_redis()
+
+    lock_acquired = False
+    lock_ttl = _sync_listing_lock_ttl_seconds()
+    heartbeat_stop: Event | None = None
+    heartbeat_thread: Thread | None = None
+
+    lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
+
+    if not lock_acquired:
+        orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id)
+        if orphan_cleared:
+            lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
+
+    if not lock_acquired:
+        logger.info("sync_listing_task skipped: another sync is already running")
+        return {
+            "status": "skipped",
+            "reason": "sync_already_running",
+            "task_id": task_id,
+        }
+
+    try:
+        full_scan_done = _is_full_scan_done(redis_client)
+        effective_only_new = False if not full_scan_done else only_new
+
+        if not full_scan_done:
+            logger.info("Bootstrap mode: forcing full scan (only_new=False) until first complete run")
+
+        # Рандомный jitter (0–120s) перед стартом.
+        jitter = random.uniform(0, 120)
+        logger.info("Anti-pattern jitter: waiting %.0fs before starting scrape", jitter)
+        time.sleep(jitter)
+
+        heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
+            redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl,
+        )
+        self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
+
+        def _job():
+            with OpenLaneScraper() as scraper:
+                return scraper.sync_listing(
+                    lane=lane,
+                    limit=limit,
+                    only_new=effective_only_new,
+                    max_pages=max_pages,
+                    concurrency=concurrency,
+                )
+
+        result = _run_browser_job(_job)
+
+        if not full_scan_done:
+            if result.get("status") == "success":
+                _set_full_scan_done(redis_client, True)
+                logger.info("Bootstrap full scan completed; hourly schedule continues")
+            else:
+                _set_full_scan_done(redis_client, False)
+
+        summary = {
+            "task_id": task_id,
+            "run_id": result.get("run_id"),
+            "status": result.get("status", "success"),
+            "cars_upserted": result.get("cars_upserted", 0),
+            "cars_failed": result.get("cars_failed", 0),
+            "images_upserted": result.get("images_upserted", 0),
+            "skipped_existing": result.get("skipped_existing", 0),
+            "elapsed_seconds": result.get("elapsed_seconds"),
+        }
+        logger.info(
+            "sync_listing_task completed: status=%s, %d upserted, %d failed",
+            summary["status"], summary["cars_upserted"], summary["cars_failed"],
+        )
+        return summary
+
+    except SoftTimeLimitExceeded:
+        logger.warning("sync_listing_task soft timeout exceeded — partial progress already saved to DB")
+        return {
+            "status": "timed_out",
+            "task_id": task_id,
+            "reason": "soft_time_limit_exceeded",
+        }
+
+    except Exception as exc:
+        logger.error("sync_listing_task failed: %s", exc, exc_info=True)
+        try:
+            raise self.retry(exc=exc)
+        except self.MaxRetriesExceededError:
+            logger.error("sync_listing_task max retries exceeded, giving up")
+            return {
+                "status": "failed",
+                "task_id": task_id,
+                "error": str(exc),
+            }
+    finally:
+        if heartbeat_stop is not None:
+            heartbeat_stop.set()
+        if heartbeat_thread is not None:
+            heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
+        if lock_acquired:
+            _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token)