initial openlane project

openlane_scraper/openlane/__init__.py

@@ -0,0 +1,4 @@
+from .mapper import map_openlane_record, map_openlane_records
+from .runner import OpenLaneScrapeRunner, OpenLaneScrapeResult
+
+__all__ = ["OpenLaneScrapeRunner", "OpenLaneScrapeResult", "map_openlane_record", "map_openlane_records"]

openlane_scraper/openlane/auth.py

@@ -0,0 +1,484 @@
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import os
+import re
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+from playwright.sync_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError
+
+from ..browser.factory import BrowserFactory
+from ..core.config import OpenLaneConfig
+
+logger = logging.getLogger("openlane_scraper.openlane.auth")
+
+# Порог предупреждения об истечении refresh_token (дни).
+_TOKEN_REFRESH_THRESHOLD_DAYS = 3
+_ACCESS_TOKEN_MIN_TTL_SECONDS = 300
+
+
+class OpenLaneAuthError(RuntimeError):
+    pass
+
+
+class OpenLaneAuthenticator:
+    def __init__(self, config: OpenLaneConfig) -> None:
+        self.config = config
+
+    def validate_credentials(self) -> None:
+        if not self.config.username or not self.config.password:
+            raise OpenLaneAuthError(
+                "OPENLANE_USERNAME and OPENLANE_PASSWORD must be set in the environment"
+            )
+
+    def interactive_login_and_persist(self, context: BrowserContext) -> str:
+        page = context.new_page()
+        page.set_default_timeout(self.config.request_timeout_ms)
+        logger.info("OpenLane interactive login: open %s and complete sign-in manually", self.config.sign_in_url)
+        page.goto(self.config.sign_in_url, wait_until="domcontentloaded")
+        self._wait_for_authenticated_session(page)
+        self._persist_storage_state(context)
+        logger.info("OpenLane interactive login successful")
+        return str(Path(self.config.storage_state_file))
+
+    def _inject_refresh_token(self, context: BrowserContext) -> None:
+        # Инъекция refresh_token cookie из env.
+        token = self.config.refresh_token
+        if not token:
+            return
+
+        # Проверка: токен уже истёк?
+        days = self._token_days_remaining(token)
+        if days is not None and days <= 0:
+            logger.warning(
+                "OPENLANE_REFRESH_TOKEN is EXPIRED (%.1f days ago). "
+                "Will attempt login with username/password.",
+                abs(days),
+            )
+            # Очищаем токен — fallback на login()
+            self.config.refresh_token = ""
+            return
+
+        if days is not None:
+            logger.info("refresh_token: %.1f days remaining (expires %s)",
+                        days, datetime.fromtimestamp(
+                            self._decode_jwt_exp(token), tz=timezone.utc  # type: ignore[arg-type]
+                        ).strftime("%Y-%m-%d %H:%M UTC"))
+            if days < _TOKEN_REFRESH_THRESHOLD_DAYS:
+                logger.warning(
+                    "refresh_token expires in %.1f days! "
+                    "Will try to re-login after auth to get a fresh one.",
+                    days,
+                )
+
+        context.add_cookies([
+            {
+                "name": "refresh_token",
+                "value": token,
+                "domain": ".openlane.com",
+                "path": "/",
+                "httpOnly": True,
+                "secure": True,
+                "sameSite": "None",
+            }
+        ])
+        logger.info("Injected OPENLANE_REFRESH_TOKEN cookie into browser context")
+
+    # ------------------------------------------------------------------
+    # Инспекция и ротация токенов
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _decode_jwt_exp(token: str) -> int | None:
+        # Декодирование JWT exp (без проверки подписи).
+        try:
+            parts = token.split(".")
+            if len(parts) < 2:
+                return None
+            payload_b64 = parts[1]
+            # Дополнение base64 padding.
+            payload_b64 += "=" * (-len(payload_b64) % 4)
+            payload = json.loads(base64.urlsafe_b64decode(payload_b64))
+            exp = payload.get("exp")
+            return int(exp) if exp is not None else None
+        except Exception:
+            return None
+
+    @staticmethod
+    def _decode_jwt_payload(token: str) -> dict[str, Any]:
+        # Декодирование JWT payload (без проверки подписи).
+        try:
+            parts = token.split(".")
+            if len(parts) < 2:
+                return {}
+            payload_b64 = parts[1]
+            payload_b64 += "=" * (-len(payload_b64) % 4)
+            payload = json.loads(base64.urlsafe_b64decode(payload_b64))
+            return payload if isinstance(payload, dict) else {}
+        except Exception:
+            return {}
+
+    @staticmethod
+    def _token_days_remaining(token: str) -> float | None:
+        # Дней до истечения токена.
+        exp = OpenLaneAuthenticator._decode_jwt_exp(token)
+        if exp is None:
+            return None
+        now = datetime.now(timezone.utc).timestamp()
+        return (exp - now) / 86400
+
+    @staticmethod
+    def _token_seconds_remaining(token: str) -> int | None:
+        # Секунд до истечения токена.
+        exp = OpenLaneAuthenticator._decode_jwt_exp(token)
+        if exp is None:
+            return None
+        return int(exp - time.time())
+
+    @staticmethod
+    def _extract_cookie_value(context: BrowserContext, name: str) -> str | None:
+        # Извлекаем значение cookie по имени.
+        cookies = context.cookies("https://app.openlane.com")
+        for cookie in cookies:
+            if cookie.get("name") == name:
+                value = str(cookie.get("value") or "").strip()
+                if value:
+                    return value
+        return None
+
+    def _is_access_token_fresh(self, token: str) -> bool:
+        # Проверяем что access_token ещё жив.
+        remain = self._token_seconds_remaining(token)
+        return remain is not None and remain > _ACCESS_TOKEN_MIN_TTL_SECONDS
+
+    def _extract_refresh_token_from_cookies(self, context: BrowserContext) -> str | None:
+        # Извлечение refresh_token из cookies.
+        return self._extract_cookie_value(context, "refresh_token")
+
+    def _extract_access_token_from_cookies(self, context: BrowserContext) -> str | None:
+        # Извлечение access_token из cookies.
+        return self._extract_cookie_value(context, "access_token")
+
+    def _refresh_access_token_via_okta(self, refresh_token: str) -> tuple[str | None, str | None]:
+        # Прямой refresh через Okta token endpoint.
+        if not refresh_token:
+            return None, None
+        if not self.config.okta_client_id:
+            logger.warning("OPENLANE_OKTA_CLIENT_ID is empty, direct refresh disabled")
+            return None, None
+
+        form_data = urlencode(
+            {
+                "grant_type": "refresh_token",
+                "client_id": self.config.okta_client_id,
+                "redirect_uri": self.config.okta_redirect_uri,
+                "refresh_token": refresh_token,
+            }
+        ).encode("utf-8")
+        req = Request(
+            self.config.okta_token_endpoint,
+            data=form_data,
+            method="POST",
+            headers={
+                "Content-Type": "application/x-www-form-urlencoded",
+                "Accept": "application/json",
+            },
+        )
+
+        try:
+            with urlopen(req, timeout=self.config.okta_timeout_seconds) as resp:
+                payload = json.loads(resp.read().decode("utf-8"))
+            access_token = str(payload.get("access_token") or "").strip()
+            new_refresh_token = str(payload.get("refresh_token") or "").strip() or refresh_token
+            if not access_token:
+                logger.warning("Okta refresh response has no access_token")
+                return None, None
+            logger.info("Okta direct refresh succeeded")
+            return access_token, new_refresh_token
+        except HTTPError as exc:
+            try:
+                body = exc.read().decode("utf-8", errors="ignore")[:300]
+            except Exception:
+                body = ""
+            logger.warning("Okta direct refresh failed: HTTP %s %s", exc.code, body)
+            return None, None
+        except URLError as exc:
+            logger.warning("Okta direct refresh failed: %s", exc)
+            return None, None
+        except Exception as exc:
+            logger.warning("Okta direct refresh failed: %s", exc)
+            return None, None
+
+    def _persist_access_token_to_storage_state(self, context: BrowserContext, access_token: str) -> None:
+        # Сохраняем access_token cookie в контекст.
+        expires = self._decode_jwt_exp(access_token)
+        if expires is None:
+            expires = int(time.time()) + 7200
+        context.add_cookies(
+            [
+                {
+                    "name": "access_token",
+                    "value": access_token,
+                    "domain": ".openlane.com",
+                    "path": "/",
+                    "httpOnly": True,
+                    "secure": True,
+                    "sameSite": "None",
+                    "expires": int(expires),
+                }
+            ]
+        )
+
+    def _maybe_rotate_token(self, context: BrowserContext) -> None:
+        # Проверка и ротация refresh_token.
+        new_token = self._extract_refresh_token_from_cookies(context)
+        if not new_token:
+            logger.debug("No refresh_token cookie found after auth — nothing to rotate")
+            return
+
+        old_token = self.config.refresh_token or ""
+
+        # Проверяем изменился ли токен.
+        if new_token != old_token:
+            days = self._token_days_remaining(new_token)
+            logger.info(
+                "refresh_token CHANGED (new exp: %.1f days). Persisting to .env",
+                days if days is not None else -1,
+            )
+            self.config.refresh_token = new_token
+            self._persist_token_to_dotenv(new_token)
+            return
+
+        # Токен не изменился — проверяем срок.
+        days = self._token_days_remaining(new_token)
+        if days is not None:
+            logger.info("refresh_token unchanged, %.1f days remaining", days)
+            if days < _TOKEN_REFRESH_THRESHOLD_DAYS:
+                logger.warning(
+                    "⚠ refresh_token expires in %.1f days! "
+                    "Run `openlane-login` or set OPENLANE_USERNAME + OPENLANE_PASSWORD "
+                    "to auto-renew on next sync.",
+                    days,
+                )
+
+    @staticmethod
+    def _persist_token_to_dotenv(token: str) -> None:
+        # Запись refresh_token в .env.
+        env_path = Path(".env")
+        if not env_path.exists():
+            # Создаём .env с токеном.
+            env_path.write_text(
+                f"OPENLANE_REFRESH_TOKEN={token}\n",
+                encoding="utf-8",
+            )
+            logger.info("Created .env with rotated OPENLANE_REFRESH_TOKEN")
+            return
+
+        content = env_path.read_text(encoding="utf-8")
+        pattern = re.compile(r"^OPENLANE_REFRESH_TOKEN=.*$", re.MULTILINE)
+        replacement = f"OPENLANE_REFRESH_TOKEN={token}"
+
+        if pattern.search(content):
+            new_content = pattern.sub(replacement, content)
+        else:
+            new_content = content.rstrip("\n") + f"\n{replacement}\n"
+
+        env_path.write_text(new_content, encoding="utf-8")
+        # Обновляем env процесса.
+        os.environ["OPENLANE_REFRESH_TOKEN"] = token
+        logger.info("Persisted rotated OPENLANE_REFRESH_TOKEN to .env")
+
+    def bootstrap_authenticated_context(self, context: BrowserContext) -> Page:
+        # Восстановление сессии: storage_state → login → ошибка.
+        storage_state_path = Path(self.config.storage_state_file)
+
+        if storage_state_path.exists():
+            logger.info("OpenLane: restoring session from %s", storage_state_path)
+            page = context.new_page()
+            page.set_default_timeout(self.config.request_timeout_ms)
+            # Блокируем трекинг/картинки — нужен только fetch().
+            BrowserFactory.enable_resource_blocking(page)
+
+            # Проверка cookies без сетевых запросов.
+            cookies = context.cookies("https://app.openlane.com")
+            cookie_names = {c["name"] for c in cookies}
+            access_token = self._extract_access_token_from_cookies(context)
+            if access_token and self._is_access_token_fresh(access_token):
+                logger.info(
+                    "OpenLane session restored: access_token cookie present (%d cookies total)",
+                    len(cookies),
+                )
+                # Лёгкий API endpoint для установки origin в браузере.
+                page.goto(
+                    "https://app.openlane.com/api/_next/time",
+                    wait_until="domcontentloaded",
+                )
+                return page
+
+            # Пробуем прямой refresh access_token без навигации.
+            refresh_token = self._extract_refresh_token_from_cookies(context) or self.config.refresh_token
+            if refresh_token:
+                new_access_token, new_refresh_token = self._refresh_access_token_via_okta(refresh_token)
+                if new_access_token:
+                    self._persist_access_token_to_storage_state(context, new_access_token)
+                    if new_refresh_token and new_refresh_token != (self.config.refresh_token or ""):
+                        self.config.refresh_token = new_refresh_token
+                        self._persist_token_to_dotenv(new_refresh_token)
+                    self._persist_storage_state(context)
+                    page.goto(
+                        "https://app.openlane.com/api/_next/time",
+                        wait_until="domcontentloaded",
+                    )
+                    logger.info("OpenLane session restored via direct Okta refresh")
+                    return page
+                logger.warning("Direct refresh failed, trying sign_in fallback")
+
+                # Если refresh_token был только в env, добавляем cookie вручную.
+                if "refresh_token" not in cookie_names and self.config.refresh_token:
+                    self._inject_refresh_token(context)
+                    cookies = context.cookies("https://app.openlane.com")
+                    cookie_names = {c["name"] for c in cookies}
+
+            logger.warning("storage_state exists but no access_token cookie — trying navigation")
+            # Fallback: SPA обменяет refresh_token на access_token.
+            if "refresh_token" in cookie_names:
+                try:
+                    page.goto(self.config.sign_in_url, wait_until="domcontentloaded")
+                    self._wait_for_authenticated_session(page)
+                    self._persist_storage_state(context)
+                    self._maybe_rotate_token(context)
+                    return page
+                except OpenLaneAuthError:
+                    logger.warning("SPA refresh_token exchange failed")
+            page.close()
+
+        # Нет storage_state — пробуем логин.
+        if self.config.username and self.config.password:
+            return self.login(context)
+
+        raise OpenLaneAuthError(
+            "No valid session found. Either:\n"
+            "  1. Run `python scripts/explore_site.py` to login manually and save storage_state, or\n"
+            "  2. Set OPENLANE_USERNAME and OPENLANE_PASSWORD in .env"
+        )
+
+    def login(self, context: BrowserContext) -> Page:
+        self.validate_credentials()
+        page = context.new_page()
+        page.set_default_timeout(self.config.request_timeout_ms)
+        logger.info("OpenLane login: opening sign-in page %s", self.config.sign_in_url)
+        page.goto(self.config.sign_in_url, wait_until="domcontentloaded")
+        self._fill_login_form(page)
+        self._wait_for_authenticated_session(page)
+        self._persist_storage_state(context)
+        self._maybe_rotate_token(context)
+        logger.info("OpenLane login successful")
+        return page
+
+    def _fill_login_form(self, page: Page) -> None:
+        email_selectors = [
+            'input[name="username"]',
+            'input[name="user[login]"]',
+            'input[name="email"]',
+            'input[name="user[email]"]',
+            'input[autocomplete="username"]',
+            'input[type="email"]',
+            'input[type="text"]',
+            '#email',
+        ]
+        password_selectors = [
+            'input[name="password"]',
+            'input[name="user[password]"]',
+            'input[type="password"]',
+            '#password',
+        ]
+        submit_selectors = [
+            'button[type="submit"]',
+            'input[type="submit"]',
+            'button:has-text("Sign in")',
+            'button:has-text("Log in")',
+        ]
+
+        email_filled = self._fill_first(page, email_selectors, self.config.username)
+        password_filled = self._fill_first(page, password_selectors, self.config.password)
+        if not email_filled or not password_filled:
+            raise OpenLaneAuthError("OpenLane login form fields were not found")
+
+        if not self._click_first(page, submit_selectors):
+            raise OpenLaneAuthError("OpenLane login submit button was not found")
+
+    def _wait_for_authenticated_session(self, page: Page) -> None:
+        # Ждём пока SPA обменяет refresh_token → access_token.
+        try:
+            page.wait_for_url(
+                lambda url: "/sign_in" not in url.lower(),
+                timeout=min(self.config.request_timeout_ms, 30000),
+            )
+        except PlaywrightTimeoutError:
+            raise OpenLaneAuthError(
+                "OpenLane authentication failed: page stayed on sign_in "
+                "(refresh_token likely invalid or expired)"
+            )
+
+        # Проверяем что не на странице ошибки.
+        current_url = page.url.lower()
+        logger.debug("After auth redirect, URL: %s", page.url)
+        if "/sign_in" in current_url:
+            raise OpenLaneAuthError("OpenLane authentication failed: still on sign_in page")
+
+        # Ждём появления access_token cookie.
+        for attempt in range(15):
+            cookies = page.context.cookies("https://app.openlane.com")
+            cookie_names = {c["name"] for c in cookies}
+            if "access_token" in cookie_names:
+                logger.info("OpenLane authenticated: access_token cookie present (attempt %d)", attempt)
+                return
+            page.wait_for_timeout(500)
+
+        # Нет access_token cookie, но URL не sign_in — продолжаем.
+        logger.warning(
+            "access_token cookie not found after redirect, but page is at %s — continuing",
+            page.url,
+        )
+
+    def _persist_storage_state(self, context: BrowserContext) -> None:
+        if not self.config.persist_storage_state:
+            return
+        storage_state_path = Path(self.config.storage_state_file)
+        storage_state_path.parent.mkdir(parents=True, exist_ok=True)
+        context.storage_state(path=str(storage_state_path))
+        logger.info("OpenLane storage state saved to %s", storage_state_path)
+
+    @staticmethod
+    def _fill_first(page: Page, selectors: list[str], value: str) -> bool:
+        for selector in selectors:
+            locator = page.locator(selector)
+            if locator.count() == 0:
+                continue
+            try:
+                locator.first.fill(value)
+                return True
+            except PlaywrightTimeoutError:
+                continue
+        return False
+
+    @staticmethod
+    def _click_first(page: Page, selectors: list[str]) -> bool:
+        for selector in selectors:
+            locator = page.locator(selector)
+            if locator.count() == 0:
+                continue
+            try:
+                locator.first.click()
+                return True
+            except PlaywrightTimeoutError:
+                continue
+        return False

openlane_scraper/openlane/checkpoint.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+
+@dataclass(slots=True)
+class OpenLaneCheckpoint:
+    max_pages: int
+    completed_pages: list[int] = field(default_factory=list)
+    failed_pages: list[int] = field(default_factory=list)
+    total_records: int = 0
+    last_saved_at: str | None = None
+
+    @property
+    def completed_set(self) -> set[int]:
+        return set(self.completed_pages)
+
+
+class OpenLaneCheckpointStore:
+    def __init__(self, path: str | Path) -> None:
+        self.path = Path(path)
+
+    def load(self, max_pages: int) -> OpenLaneCheckpoint:
+        if not self.path.exists():
+            return OpenLaneCheckpoint(max_pages=max_pages)
+        data = json.loads(self.path.read_text(encoding="utf-8"))
+        return OpenLaneCheckpoint(
+            max_pages=int(data.get("max_pages") or max_pages),
+            completed_pages=sorted({int(page) for page in data.get("completed_pages", [])}),
+            failed_pages=sorted({int(page) for page in data.get("failed_pages", [])}),
+            total_records=int(data.get("total_records") or 0),
+            last_saved_at=data.get("last_saved_at"),
+        )
+
+    def save(self, checkpoint: OpenLaneCheckpoint) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        payload = asdict(checkpoint)
+        self.path.write_text(
+            json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
+            encoding="utf-8",
+        )

openlane_scraper/openlane/client.py

@@ -0,0 +1,376 @@
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import random
+import time
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import urlencode
+
+from playwright.sync_api import Page
+
+from ..core.config import OpenLaneConfig
+
+logger = logging.getLogger("openlane_scraper.openlane.client")
+
+
+class OpenLaneRequestError(RuntimeError):
+    def __init__(self, message: str, *, status_code: int | None = None) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+
+
+@dataclass(slots=True)
+class OpenLanePageResult:
+    page: int
+    records: list[dict[str, Any]]
+    raw_payload: dict[str, Any]
+    status_code: int
+
+
+def _decode_access_token(token: str) -> dict[str, Any]:
+    # Декодирование JWT payload (без проверки подписи).
+    parts = token.split(".")
+    if len(parts) < 2:
+        return {}
+    payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4)
+    try:
+        return json.loads(base64.urlsafe_b64decode(payload_b64))
+    except Exception:
+        return {}
+
+
+class OpenLaneClient:
+    # HTTP-клиент OpenLane search API через Playwright.
+
+    def __init__(self, authenticated_page: Page, config: OpenLaneConfig) -> None:
+        self.page = authenticated_page
+        self.config = config
+        self._user_id: str | None = None
+        self._dealership_id: str | None = None
+        self._init_ids_from_cookies()
+
+    def _init_ids_from_cookies(self) -> None:
+        # Извлекаем user_id и dealership_id из access_token.
+        cookies = self.page.context.cookies("https://app.openlane.com")
+        for cookie in cookies:
+            if cookie.get("name") == "access_token":
+                payload = _decode_access_token(cookie["value"])
+                self._user_id = str(payload.get("user_id", ""))
+                on_behalf = payload.get("on_behalf_of", {})
+                self._dealership_id = str(on_behalf.get("dealership_id", ""))
+                if self._user_id and self._dealership_id:
+                    logger.info(
+                        "OpenLane client: user_id=%s dealership_id=%s",
+                        self._user_id, self._dealership_id,
+                    )
+                return
+        logger.warning("OpenLane client: access_token cookie not found, API requests may fail")
+
+    def _build_headers(self) -> dict[str, str]:
+        # Заголовки как у фронтенда OpenLane.
+        headers: dict[str, str] = {
+            "Accept": "application/json, application/vnd.backlotcars.v3",
+            "application": "webapp",
+        }
+        if self._user_id:
+            headers["x-rbz-user-id"] = self._user_id
+        if self._dealership_id:
+            headers["x-rbz-dealership-id"] = self._dealership_id
+        return headers
+
+    def fetch_page(self, page: int) -> OpenLanePageResult:
+        params = {
+            "page": page,
+            "source_tab": self.config.source_tab,
+            "sale_types": self.config.sale_types,
+        }
+        if self.config.page_size > 0:
+            params["per_page"] = self.config.page_size
+
+        self._sleep_with_jitter()
+        url = f"{self.config.api_search_url}?{urlencode(params)}"
+        logger.debug("OpenLane fetch page=%s url=%s", page, url)
+
+        # fetch() с авторизацией в браузере.
+        headers_js = json.dumps(self._build_headers())
+        fetch_result = self.page.evaluate(
+            """async ([url, headersJson]) => {
+                const headers = JSON.parse(headersJson);
+                const resp = await fetch(url, {
+                    method: 'GET',
+                    credentials: 'include',
+                    headers: headers
+                });
+                const text = await resp.text();
+                return { status: resp.status, body: text };
+            }""",
+            [url, headers_js],
+        )
+        return self._parse_response(page, fetch_result)
+
+    def fetch_vehicle_images(self, vehicle_id: str | int) -> list[dict[str, Any]]:
+        """Загружает изображения автомобиля через `/api/vehicles/{id}/images`."""
+        if vehicle_id is None:
+            return []
+
+        vehicle_id_str = str(vehicle_id).strip()
+        if not vehicle_id_str:
+            return []
+
+        self._sleep_with_jitter()
+        url = f"https://app.openlane.com/api/vehicles/{vehicle_id_str}/images"
+        logger.debug("OpenLane fetch images vehicle_id=%s", vehicle_id_str)
+
+        headers_js = json.dumps(self._build_headers())
+        fetch_result = self.page.evaluate(
+            """async ([url, headersJson]) => {
+                const headers = JSON.parse(headersJson);
+                const resp = await fetch(url, {
+                    method: 'GET',
+                    credentials: 'include',
+                    headers: headers
+                });
+                const text = await resp.text();
+                return { status: resp.status, body: text };
+            }""",
+            [url, headers_js],
+        )
+
+        status_code = int(fetch_result.get("status", 0))
+        text = str(fetch_result.get("body", ""))
+        if status_code == 404:
+            return []
+        if status_code == 403:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
+                status_code=status_code,
+            )
+        if status_code == 429:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
+                status_code=status_code,
+            )
+        if status_code >= 500:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
+                status_code=status_code,
+            )
+        if status_code == 401:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
+                status_code=status_code,
+            )
+        if status_code >= 400:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
+                status_code=status_code,
+            )
+
+        if not text.strip():
+            return []
+
+        try:
+            payload = json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise OpenLaneRequestError(
+                f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
+            ) from exc
+
+        return self._extract_vehicle_images(payload)
+
+    def fetch_vehicle_images_batch(self, vehicle_ids: list[str | int]) -> dict[str, list[dict[str, Any]]]:
+        """Пакетно загружает изображения автомобилей через `/api/vehicles/{id}/images`.
+
+        Возвращает словарь `vehicle_id -> список image-объектов`.
+        """
+        normalized_ids: list[str] = []
+        seen: set[str] = set()
+        for vehicle_id in vehicle_ids:
+            vehicle_id_str = str(vehicle_id).strip()
+            if not vehicle_id_str or vehicle_id_str in seen:
+                continue
+            seen.add(vehicle_id_str)
+            normalized_ids.append(vehicle_id_str)
+
+        if not normalized_ids:
+            return {}
+
+        self._sleep_with_jitter()
+        headers_js = json.dumps(self._build_headers())
+        fetch_results = self.page.evaluate(
+            """async ([vehicleIds, headersJson]) => {
+                const headers = JSON.parse(headersJson);
+                const tasks = vehicleIds.map(async (vehicleId) => {
+                    try {
+                        const url = `https://app.openlane.com/api/vehicles/${vehicleId}/images`;
+                        const resp = await fetch(url, {
+                            method: 'GET',
+                            credentials: 'include',
+                            headers: headers,
+                        });
+                        const text = await resp.text();
+                        return { vehicleId, status: resp.status, body: text };
+                    } catch (error) {
+                        return { vehicleId, status: 0, body: '', error: String(error) };
+                    }
+                });
+                return await Promise.all(tasks);
+            }""",
+            [normalized_ids, headers_js],
+        )
+
+        result_map: dict[str, list[dict[str, Any]]] = {}
+        for item in fetch_results:
+            vehicle_id_str = str(item.get("vehicleId", "")).strip()
+            status_code = int(item.get("status", 0))
+            text = str(item.get("body", ""))
+
+            if not vehicle_id_str:
+                continue
+
+            if status_code == 404:
+                result_map[vehicle_id_str] = []
+                continue
+            if status_code == 403:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden",
+                    status_code=status_code,
+                )
+            if status_code == 429:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests",
+                    status_code=status_code,
+                )
+            if status_code >= 500 or status_code == 0:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}",
+                    status_code=status_code if status_code else 500,
+                )
+            if status_code == 401:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated",
+                    status_code=status_code,
+                )
+            if status_code >= 400:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}",
+                    status_code=status_code,
+                )
+
+            if not text.strip():
+                result_map[vehicle_id_str] = []
+                continue
+
+            try:
+                payload = json.loads(text)
+            except json.JSONDecodeError as exc:
+                raise OpenLaneRequestError(
+                    f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON"
+                ) from exc
+
+            result_map[vehicle_id_str] = self._extract_vehicle_images(payload)
+
+        return result_map
+
+    def _parse_response(self, page: int, fetch_result: dict[str, Any]) -> OpenLanePageResult:
+        # Парсинг ответа fetch(): {status, body}.
+        status_code = int(fetch_result.get("status", 0))
+        text = str(fetch_result.get("body", ""))
+        logger.debug(
+            "OpenLane response: page=%s status=%s body_len=%d body_preview=%.500s",
+            page, status_code, len(text), text[:500],
+        )
+        if status_code == 403:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned 403 Forbidden — possible block, stopping",
+                status_code=status_code,
+            )
+        if status_code == 429:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned 429 Too Many Requests — rate limited",
+                status_code=status_code,
+            )
+        if status_code >= 500:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned server error {status_code}",
+                status_code=status_code,
+            )
+        if status_code == 401:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned 401; session is not authenticated",
+                status_code=status_code,
+            )
+        if status_code >= 400:
+            raise OpenLaneRequestError(
+                f"OpenLane page={page} returned unexpected status {status_code}",
+                status_code=status_code,
+            )
+
+        try:
+            payload = json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise OpenLaneRequestError(f"OpenLane page={page} returned invalid JSON") from exc
+
+        records = self._extract_records(payload)
+        logger.debug(
+            "OpenLane extracted: page=%s records=%d top_keys=%s",
+            page, len(records), list(payload.keys())[:10],
+        )
+        return OpenLanePageResult(
+            page=page,
+            records=records,
+            raw_payload=payload,
+            status_code=status_code,
+        )
+
+    @staticmethod
+    def _extract_records(payload: dict[str, Any]) -> list[dict[str, Any]]:
+        candidates = [
+            payload.get("results"),
+            payload.get("items"),
+            payload.get("data"),
+            payload.get("vehicles"),
+        ]
+        for candidate in candidates:
+            if isinstance(candidate, list):
+                return [item for item in candidate if isinstance(item, dict)]
+            if isinstance(candidate, dict):
+                nested_candidates = [
+                    candidate.get("results"),
+                    candidate.get("items"),
+                    candidate.get("vehicles"),
+                ]
+                for nested in nested_candidates:
+                    if isinstance(nested, list):
+                        return [item for item in nested if isinstance(item, dict)]
+        return []
+
+    @staticmethod
+    def _extract_vehicle_images(payload: Any) -> list[dict[str, Any]]:
+        candidates: list[Any] = []
+        if isinstance(payload, dict):
+            candidates.extend(
+                [
+                    payload.get("vehicle_images"),
+                    payload.get("images"),
+                    payload.get("data"),
+                ]
+            )
+            data = payload.get("data")
+            if isinstance(data, dict):
+                candidates.extend([data.get("vehicle_images"), data.get("images")])
+        elif isinstance(payload, list):
+            candidates.append(payload)
+
+        for candidate in candidates:
+            if isinstance(candidate, list):
+                return [item for item in candidate if isinstance(item, dict)]
+        return []
+
+    def _sleep_with_jitter(self) -> None:
+        low = min(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
+        high = max(self.config.throttle_min_seconds, self.config.throttle_max_seconds)
+        time.sleep(random.uniform(low, high))

openlane_scraper/openlane/mapper.py

@@ -0,0 +1,424 @@
+"""Маппер: JSON-запись OpenLane API → CarRecord для сохранения в БД."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from datetime import datetime, timezone
+from typing import Any
+
+from ..storage.schemas import CarRecord, ImageRecord
+
+logger = logging.getLogger("openlane_scraper.openlane.mapper")
+
+# Маппинг body_type из OpenLane в наш enum.
+BODY_MAP: dict[str, str] = {
+    "sedan": "SEDAN",
+    "coupe": "COUPE",
+    "suv": "SUV",
+    "sport utility": "SUV",
+    "crossover": "SUV",
+    "hatchback": "HATCHBACK",
+    "minivan": "MINIVAN",
+    "van": "MINIVAN",
+    "wagon": "STATION_WAGON",
+    "station wagon": "STATION_WAGON",
+    "pickup": "PICKUP",
+    "truck": "TRUCK",
+    "convertible": "OPEN",
+    "cabriolet": "OPEN",
+    "roadster": "OPEN",
+    "rv": "RV",
+    "motorhome": "RV",
+}
+
+DRIVE_MAP: dict[str, str] = {
+    "fwd": "FWD",
+    "front wheel drive": "FWD",
+    "front-wheel drive": "FWD",
+    "rwd": "RWD",
+    "rear wheel drive": "RWD",
+    "rear-wheel drive": "RWD",
+    "awd": "4WD",
+    "4wd": "4WD",
+    "all wheel drive": "4WD",
+    "all-wheel drive": "4WD",
+    "4x4": "4WD",
+    "2wd": "2WD",
+    "two wheel drive": "2WD",
+}
+
+GEARBOX_MAP: dict[str, str] = {
+    "automatic": "AT",
+    "auto": "AT",
+    "at": "AT",
+    "manual": "MT",
+    "mt": "MT",
+    "cvt": "CVT",
+    "continuously variable": "CVT",
+    "electric": "EV",
+    "ev": "EV",
+}
+
+_MILEAGE_RE = re.compile(r"[\d,]+")
+_ENGINE_RE = re.compile(r"(\d+\.?\d*)\s*[lL]")
+
+
+def _safe_str(value: Any, default: str = "") -> str:
+    if value is None:
+        return default
+    return str(value).strip() or default
+
+
+def _safe_int(value: Any) -> int | None:
+    if value is None:
+        return None
+    try:
+        cleaned = str(value).replace(",", "").strip()
+        if not cleaned:
+            return None
+        return int(float(cleaned))
+    except (ValueError, TypeError):
+        return None
+
+
+def _normalize_enum(raw: Any, mapping: dict[str, str], default: str = "NA") -> str:
+    if not raw:
+        return default
+    key = str(raw).strip().lower()
+    return mapping.get(key, default)
+
+
+def _generate_parser_id(origin_id: str) -> str:
+    return hashlib.sha256(origin_id.encode()).hexdigest()[:40]
+
+
+def _generate_slug(year: int | None, brand: str, model: str, origin_id: str) -> str:
+    parts = []
+    if year:
+        parts.append(str(year))
+    parts.append(brand.lower())
+    parts.append(model.lower())
+    parts.append(origin_id.replace(":", "-"))
+    slug = "-".join(parts)
+    slug = re.sub(r"[^a-z0-9\-]", "-", slug)
+    slug = re.sub(r"-+", "-", slug).strip("-")
+    return slug[:200]
+
+
+def _build_openlane_origin_id(raw_id: Any) -> str | None:
+    """Строит origin_id в формате openlane:id."""
+    normalized = _safe_str(raw_id)
+    if not normalized:
+        return None
+    if normalized.lower().startswith("openlane:"):
+        normalized = _safe_str(normalized.split(":", 1)[1])
+    if not normalized:
+        return None
+    return f"openlane:{normalized}"
+
+
+def _extract_nested(record: dict[str, Any], *keys: str) -> Any:
+    """Извлекает значение из вложенного словаря по цепочке ключей."""
+    obj: Any = record
+    for key in keys:
+        if isinstance(obj, dict):
+            obj = obj.get(key)
+        else:
+            return None
+    return obj
+
+
+def _extract_images(record: dict[str, Any]) -> list[ImageRecord]:
+    """Извлекает изображения из записи OpenLane."""
+    images: list[ImageRecord] = []
+    seen_urls: set[str] = set()
+
+    raw_images = (
+        record.get("images")
+        or record.get("photos")
+        or record.get("media", {}).get("images")
+        or record.get("image_urls")
+        or []
+    )
+
+    if isinstance(raw_images, list):
+        for idx, img in enumerate(raw_images):
+            if isinstance(img, str):
+                url = img.strip()
+                if url and url not in seen_urls:
+                    seen_urls.add(url)
+                    images.append(ImageRecord(
+                        fullres_image=url,
+                        preview_image=url,
+                        order_index=idx,
+                    ))
+            elif isinstance(img, dict):
+                fullres = _safe_str(
+                    img.get("full") or img.get("fullres") or img.get("url")
+                    or img.get("original") or img.get("large") or img.get("href")
+                    or img.get("large_resolution_url")
+                )
+                preview = _safe_str(
+                    img.get("thumbnail") or img.get("thumb") or img.get("preview")
+                    or img.get("small") or img.get("low_resolution_url")
+                    or fullres
+                )
+                if fullres and fullres not in seen_urls:
+                    seen_urls.add(fullres)
+                    images.append(ImageRecord(
+                        fullres_image=fullres,
+                        preview_image=preview,
+                        order_index=idx,
+                    ))
+
+    # Fallback: одиночное изображение.
+    if not images:
+        single = (
+            record.get("image_url")
+            or record.get("image")
+            or record.get("large_resolution_url")
+            or record.get("low_resolution_url")
+            or record.get("thumbnail")
+            or record.get("photo_url")
+            or _extract_nested(record, "media", "primary")
+        )
+        if single and isinstance(single, str) and single.strip():
+            images.append(ImageRecord(
+                fullres_image=single.strip(),
+                preview_image=single.strip(),
+                order_index=0,
+            ))
+
+    return images
+
+
+def _parse_mileage(raw: Any) -> int:
+    if raw is None:
+        return 0
+    if isinstance(raw, (int, float)):
+        return max(0, int(raw))
+    text = str(raw)
+    match = _MILEAGE_RE.search(text)
+    if match:
+        try:
+            return max(0, int(match.group().replace(",", "")))
+        except ValueError:
+            pass
+    return 0
+
+
+def _parse_engine_volume_cc(raw: Any) -> int | None:
+    """Парсит объём двигателя и возвращает значение в кубических сантиметрах."""
+    if raw is None:
+        return None
+    if isinstance(raw, (int, float)):
+        value = float(raw)
+        # Если значение < 20 — скорее всего это литры, конвертируем в cc.
+        if 0 < value < 20:
+            return int(value * 1000)
+        if value >= 100:
+            return int(value)
+        return None
+    text = str(raw)
+    match = _ENGINE_RE.search(text)
+    if match:
+        liters = float(match.group(1))
+        return int(liters * 1000)
+    return None
+
+
+def map_openlane_record(record: dict[str, Any]) -> CarRecord | None:
+    """Маппит одну запись из OpenLane API в CarRecord.
+
+    Возвращает None, если запись не содержит минимально необходимых данных.
+    """
+    # Извлекаем идентификатор.
+    raw_id = (
+        record.get("id")
+        or record.get("vehicle_id")
+        or record.get("listing_id")
+        or record.get("vin")
+    )
+    if not raw_id:
+        logger.debug("Skipping record without id: %s", record.get("vin", "unknown"))
+        return None
+
+    origin_id = _build_openlane_origin_id(raw_id)
+    if not origin_id:
+        logger.debug("Skipping record with malformed id: %s", raw_id)
+        return None
+
+    # Бренд и модель — обязательные поля.
+    brand = _safe_str(
+        record.get("make")
+        or record.get("brand")
+        or record.get("manufacturer")
+        or _extract_nested(record, "vehicle", "make")
+    ).upper()
+
+    model = _safe_str(
+        record.get("model")
+        or record.get("model_name")
+        or _extract_nested(record, "vehicle", "model")
+    ).upper()
+
+    if not brand or not model:
+        logger.debug("Skipping record without brand/model: %s", origin_id)
+        return None
+
+    year = _safe_int(
+        record.get("year")
+        or record.get("model_year")
+        or _extract_nested(record, "vehicle", "year")
+    )
+
+    price = _safe_int(
+        record.get("price")
+        or record.get("current_bid")
+        or record.get("buy_now_price")
+        or record.get("asking_price")
+        or record.get("sale_price")
+        or _extract_nested(record, "pricing", "current")
+        or _extract_nested(record, "pricing", "buy_now")
+    )
+
+    currency = _safe_str(
+        record.get("currency")
+        or record.get("currency_code")
+        or _extract_nested(record, "pricing", "currency"),
+        "USD",
+    ).upper()
+    if currency not in {"JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD"}:
+        currency = "USD"
+
+    mileage = _parse_mileage(
+        record.get("mileage")
+        or record.get("odometer")
+        or record.get("odometer_reading")
+        or _extract_nested(record, "vehicle", "mileage")
+    )
+
+    color = _safe_str(
+        record.get("color")
+        or record.get("exterior_color")
+        or _extract_nested(record, "vehicle", "color"),
+        "other",
+    ).lower()
+
+    body_type = _normalize_enum(
+        record.get("body_type")
+        or record.get("body_style")
+        or record.get("vehicle_type")
+        or _extract_nested(record, "vehicle", "body_type"),
+        BODY_MAP,
+        "OTHER",
+    )
+
+    drive = _normalize_enum(
+        record.get("drive_type")
+        or record.get("drivetrain")
+        or record.get("drive")
+        or _extract_nested(record, "vehicle", "drivetrain"),
+        DRIVE_MAP,
+    )
+
+    gearbox = _normalize_enum(
+        record.get("transmission")
+        or record.get("gearbox")
+        or _extract_nested(record, "vehicle", "transmission"),
+        GEARBOX_MAP,
+    )
+
+    engine_volume = _parse_engine_volume_cc(
+        record.get("engine")
+        or record.get("engine_size")
+        or record.get("displacement")
+        or _extract_nested(record, "vehicle", "engine")
+    )
+
+    # URL записи на OpenLane.
+    origin_url = _safe_str(
+        record.get("url")
+        or record.get("listing_url")
+        or record.get("detail_url")
+        or record.get("permalink")
+    )
+    if not origin_url:
+        origin_url = f"https://app.openlane.com/vehicles/{raw_id}"
+
+    country = _safe_str(
+        record.get("country")
+        or record.get("location_country")
+        or _extract_nested(record, "location", "country"),
+        "US",
+    ).upper()
+    if country not in {"JP", "KR", "US", "CA", "NA"}:
+        country = "US"
+
+    is_damaged = bool(
+        record.get("is_damaged")
+        or record.get("has_damage")
+        or record.get("damage_type")
+    )
+
+    vin = _safe_str(record.get("vin") or _extract_nested(record, "vehicle", "vin"))
+    evaluation = vin if vin else None
+
+    selling_type = "AUCTION"
+    sale_type = _safe_str(record.get("sale_type") or record.get("listing_type")).lower()
+    if sale_type in {"buy_now", "fixed_price", "stock"}:
+        selling_type = "STOCK"
+    elif sale_type in {"tender"}:
+        selling_type = "TENDER"
+
+    images = _extract_images(record)
+
+    parser_id = _generate_parser_id(origin_id)
+    slug = _generate_slug(year, brand, model, origin_id)
+
+    return CarRecord(
+        parser_id=parser_id,
+        brand=brand,
+        model=model,
+        year=year,
+        price=price,
+        currency=currency,
+        mileage=mileage,
+        country=country,
+        is_sold=False,
+        color=color,
+        drive=drive if drive != "NA" else None,
+        gearbox=gearbox if gearbox != "NA" else None,
+        body_type=body_type,
+        engine_volume=engine_volume,
+        selling_type=selling_type,
+        origin="OPENLANE",
+        origin_url=origin_url,
+        origin_id=origin_id,
+        is_damaged=is_damaged,
+        evaluation=evaluation,
+        slug=slug,
+        images=images,
+    )
+
+
+def map_openlane_records(records: list[dict[str, Any]]) -> list[CarRecord]:
+    """Маппит список записей OpenLane API в список CarRecord.
+
+    Пропускает записи без минимально необходимых данных.
+    """
+    result: list[CarRecord] = []
+    for record in records:
+        try:
+            car = map_openlane_record(record)
+            if car is not None:
+                result.append(car)
+        except Exception:
+            logger.warning(
+                "Failed to map OpenLane record id=%s",
+                record.get("id", "unknown"),
+                exc_info=True,
+            )
+    return result

openlane_scraper/openlane/runner.py

@@ -0,0 +1,273 @@
+from __future__ import annotations
+
+import logging
+import time
+import uuid
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from playwright.sync_api import sync_playwright
+
+from ..browser.factory import BrowserFactory
+from ..core.config import Settings
+from ..core.logs import set_trace_id, setup_logging
+from .auth import OpenLaneAuthenticator
+from .checkpoint import OpenLaneCheckpoint, OpenLaneCheckpointStore
+from .client import OpenLaneClient, OpenLanePageResult, OpenLaneRequestError
+from .writer import OpenLaneResultWriter
+
+logger = logging.getLogger("openlane_scraper.openlane.runner")
+
+
+@dataclass(slots=True)
+class OpenLaneScrapeResult:
+    jsonl_path: str
+    aggregated_path: str
+    checkpoint_path: str
+    storage_state_path: str
+    completed_pages: list[int]
+    failed_pages: list[int]
+    total_records: int
+    elapsed_seconds: float
+
+
+class OpenLaneScrapeRunner:
+    def __init__(self, runtime_settings: Settings | None = None) -> None:
+        self.settings = runtime_settings or Settings()
+        setup_logging(self.settings.log_level, self.settings.log_file)
+        self.trace_id = f"openlane-{uuid.uuid4().hex[:8]}"
+        set_trace_id(self.trace_id)
+        self.openlane = self.settings.openlane
+        self.browser_factory = BrowserFactory(self.settings)
+        self.authenticator = OpenLaneAuthenticator(self.openlane)
+        self.writer = OpenLaneResultWriter(
+            self.openlane.jsonl_output,
+            self.openlane.aggregated_output,
+        )
+        self.checkpoint_store = OpenLaneCheckpointStore(self.openlane.checkpoint_file)
+
+    def run(
+        self,
+        *,
+        max_pages: int | None = None,
+        concurrency: int | None = None,
+        resume: bool = False,
+        checkpoint_every_pages: int | None = None,
+    ) -> OpenLaneScrapeResult:
+        started_at = time.perf_counter()
+        max_pages = max_pages or self.openlane.max_pages
+        concurrency = max(1, min(concurrency or self.openlane.concurrency, 5))
+        checkpoint_every_pages = checkpoint_every_pages or self.openlane.checkpoint_every_pages
+
+        checkpoint = self.checkpoint_store.load(max_pages=max_pages) if resume else OpenLaneCheckpoint(max_pages=max_pages)
+        checkpoint.max_pages = max_pages
+
+        if not resume:
+            self._reset_outputs()
+
+        logger.info(
+            "Starting OpenLane scrape: max_pages=%s concurrency=%s resume=%s checkpoint_every_pages=%s",
+            max_pages,
+            concurrency,
+            resume,
+            checkpoint_every_pages,
+        )
+
+        with sync_playwright() as playwright:
+            browser = self.browser_factory.create_browser(playwright)
+            try:
+                auth_pages = []
+                for worker_index in range(concurrency):
+                    storage_state_path = self.openlane.storage_state_file if Path(self.openlane.storage_state_file).exists() else None
+                    context = self.browser_factory.create_context(browser, storage_state_path=storage_state_path)
+                    auth_page = self.authenticator.bootstrap_authenticated_context(context)
+                    auth_pages.append(auth_page)
+                    logger.info("OpenLane worker session initialized: worker=%s", worker_index + 1)
+
+                pending_pages = [
+                    page for page in range(1, max_pages + 1)
+                    if page not in checkpoint.completed_set
+                ]
+                self._run_workers(auth_pages, pending_pages, checkpoint, checkpoint_every_pages)
+            finally:
+                browser.close()
+
+        self.checkpoint_store.save(checkpoint)
+        aggregated = self.writer.finalize(
+            max_pages=max_pages,
+            completed_pages=checkpoint.completed_pages,
+            failed_pages=checkpoint.failed_pages,
+        )
+        elapsed_seconds = time.perf_counter() - started_at
+        logger.info(
+            "OpenLane scrape finished: completed_pages=%s failed_pages=%s total_records=%s elapsed=%.2fs",
+            len(set(checkpoint.completed_pages)),
+            len(set(checkpoint.failed_pages)),
+            aggregated["total_records"],
+            elapsed_seconds,
+        )
+        return OpenLaneScrapeResult(
+            jsonl_path=str(Path(self.openlane.jsonl_output)),
+            aggregated_path=str(Path(self.openlane.aggregated_output)),
+            checkpoint_path=str(Path(self.openlane.checkpoint_file)),
+            storage_state_path=str(Path(self.openlane.storage_state_file)),
+            completed_pages=sorted(set(checkpoint.completed_pages)),
+            failed_pages=sorted(set(checkpoint.failed_pages)),
+            total_records=int(aggregated["total_records"]),
+            elapsed_seconds=elapsed_seconds,
+        )
+
+    def _run_workers(
+        self,
+        auth_pages: list,
+        pending_pages: list[int],
+        checkpoint: OpenLaneCheckpoint,
+        checkpoint_every_pages: int,
+    ) -> None:
+        # Последовательная обработка страниц (sync API — однопоточный).
+        started_at = time.perf_counter()
+
+        for idx, page_num in enumerate(pending_pages):
+            worker_index = (idx % len(auth_pages)) + 1
+            auth_page = auth_pages[idx % len(auth_pages)]
+
+            try:
+                result = self._fetch_page_with_retry(auth_page, page_num, worker_index)
+                self._handle_success(result, checkpoint, started_at)
+            except Exception as exc:
+                self._handle_failure(page_num, exc, checkpoint, started_at)
+
+            processed_count = len(set(checkpoint.completed_pages)) + len(set(checkpoint.failed_pages))
+            if processed_count and processed_count % checkpoint_every_pages == 0:
+                checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat()
+                self.checkpoint_store.save(checkpoint)
+                logger.info(
+                    "OpenLane checkpoint saved: processed=%s completed=%s failed=%s",
+                    processed_count,
+                    len(set(checkpoint.completed_pages)),
+                    len(set(checkpoint.failed_pages)),
+                )
+
+    def _fetch_page_with_retry(self, authenticated_page, page: int, worker_index: int) -> OpenLanePageResult:
+        set_trace_id(f"{self.trace_id}-w{worker_index}-p{page}")
+        client = OpenLaneClient(authenticated_page, self.openlane)
+        retry_schedule = self.openlane.retry_schedule_seconds
+
+        for attempt in range(len(retry_schedule) + 1):
+            try:
+                logger.debug("OpenLane fetch attempt: page=%s worker=%s attempt=%s", page, worker_index, attempt + 1)
+                return client.fetch_page(page)
+            except OpenLaneRequestError as exc:
+                is_retryable = exc.status_code in {403, 429} or (exc.status_code is not None and exc.status_code >= 500)
+                if not is_retryable or attempt >= len(retry_schedule):
+                    logger.error(
+                        "OpenLane fetch failed permanently: page=%s worker=%s status=%s error=%s",
+                        page,
+                        worker_index,
+                        exc.status_code,
+                        exc,
+                    )
+                    raise
+                delay = retry_schedule[attempt]
+                logger.warning(
+                    "OpenLane retry scheduled: page=%s worker=%s status=%s delay=%.1fs attempt=%s",
+                    page,
+                    worker_index,
+                    exc.status_code,
+                    delay,
+                    attempt + 1,
+                )
+                time.sleep(delay)
+            except Exception:
+                if attempt >= len(retry_schedule):
+                    raise
+                delay = retry_schedule[attempt]
+                logger.warning(
+                    "OpenLane transient error retry: page=%s worker=%s delay=%.1fs attempt=%s",
+                    page,
+                    worker_index,
+                    delay,
+                    attempt + 1,
+                    exc_info=True,
+                )
+                time.sleep(delay)
+
+        raise RuntimeError(f"OpenLane page {page} exhausted retries")
+
+    def _handle_success(
+        self,
+        result: OpenLanePageResult,
+        checkpoint: OpenLaneCheckpoint,
+        started_at: float,
+    ) -> None:
+        written = self.writer.append_page(result.page, result.records)
+        checkpoint.completed_pages = sorted(set(checkpoint.completed_pages) | {result.page})
+        checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) - {result.page})
+        checkpoint.total_records += written
+        checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat()
+        self._log_progress(result.page, checkpoint, started_at, written, None)
+
+    def _handle_failure(
+        self,
+        page: int,
+        exc: Exception,
+        checkpoint: OpenLaneCheckpoint,
+        started_at: float,
+    ) -> None:
+        checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) | {page})
+        checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat()
+        self._log_progress(page, checkpoint, started_at, 0, exc)
+
+    def _log_progress(
+        self,
+        page: int,
+        checkpoint: OpenLaneCheckpoint,
+        started_at: float,
+        records_written: int,
+        exc: Exception | None,
+    ) -> None:
+        completed = len(set(checkpoint.completed_pages))
+        failed = len(set(checkpoint.failed_pages))
+        elapsed_seconds = max(time.perf_counter() - started_at, 0.001)
+        pages_per_minute = (completed + failed) / elapsed_seconds * 60.0
+        if exc is None:
+            logger.info(
+                "OpenLane progress: page=%s completed=%s failed=%s records=%s total_records=%s speed=%.2f pages/min",
+                page,
+                completed,
+                failed,
+                records_written,
+                checkpoint.total_records,
+                pages_per_minute,
+            )
+        else:
+            logger.error(
+                "OpenLane page error: page=%s completed=%s failed=%s total_records=%s speed=%.2f pages/min error=%s",
+                page,
+                completed,
+                failed,
+                checkpoint.total_records,
+                pages_per_minute,
+                exc,
+            )
+
+    def interactive_login(self) -> str:
+        setup_logging(self.settings.log_level, self.settings.log_file)
+        with sync_playwright() as playwright:
+            browser = self.browser_factory.create_browser(playwright)
+            try:
+                context = self.browser_factory.create_context(browser)
+                return self.authenticator.interactive_login_and_persist(context)
+            finally:
+                browser.close()
+
+    def _reset_outputs(self) -> None:
+        for raw_path in (
+            self.openlane.jsonl_output,
+            self.openlane.aggregated_output,
+            self.openlane.checkpoint_file,
+        ):
+            path = Path(raw_path)
+            if path.exists():
+                path.unlink()

openlane_scraper/openlane/writer.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+class OpenLaneResultWriter:
+    def __init__(self, jsonl_path: str | Path, aggregated_path: str | Path) -> None:
+        self.jsonl_path = Path(jsonl_path)
+        self.aggregated_path = Path(aggregated_path)
+
+    def ensure_parent_dirs(self) -> None:
+        self.jsonl_path.parent.mkdir(parents=True, exist_ok=True)
+        self.aggregated_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def append_page(self, page: int, records: list[dict[str, Any]]) -> int:
+        self.ensure_parent_dirs()
+        written = 0
+        with self.jsonl_path.open("a", encoding="utf-8") as fh:
+            for record in records:
+                payload = {
+                    "page": page,
+                    "record": record,
+                }
+                fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
+                written += 1
+        return written
+
+    def finalize(self, *, max_pages: int, completed_pages: list[int], failed_pages: list[int]) -> dict[str, Any]:
+        self.ensure_parent_dirs()
+        records: list[dict[str, Any]] = []
+        if self.jsonl_path.exists():
+            with self.jsonl_path.open("r", encoding="utf-8") as fh:
+                for line in fh:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    item = json.loads(line)
+                    records.append(item)
+
+        summary = {
+            "max_pages": max_pages,
+            "completed_pages": sorted(completed_pages),
+            "failed_pages": sorted(failed_pages),
+            "total_pages_completed": len(set(completed_pages)),
+            "total_pages_failed": len(set(failed_pages)),
+            "total_records": len(records),
+            "items": records,
+        }
+        self.aggregated_path.write_text(
+            json.dumps(summary, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        return summary