From 74f12da10a9d5487559d8338098928990df313dc Mon Sep 17 00:00:00 2001 From: qananasikq Date: Tue, 21 Apr 2026 23:49:56 +0300 Subject: [PATCH] add openlane scraper --- openlane_scraper/__init__.py | 1 + openlane_scraper/api/__init__.py | 3 + openlane_scraper/api/app.py | 40 ++ openlane_scraper/api/deps.py | 9 + openlane_scraper/api/routes/__init__.py | 0 openlane_scraper/api/routes/cars.py | 103 +++++ openlane_scraper/api/routes/health.py | 30 ++ openlane_scraper/api/routes/tasks.py | 96 ++++ openlane_scraper/browser/__init__.py | 3 + openlane_scraper/browser/factory.py | 226 ++++++++++ openlane_scraper/cli.py | 92 ++++ openlane_scraper/core/__init__.py | 1 + openlane_scraper/core/config.py | 194 ++++++++ openlane_scraper/core/logs.py | 39 ++ openlane_scraper/core/runtime_config.py | 379 ++++++++++++++++ openlane_scraper/core/utils.py | 66 +++ openlane_scraper/openlane/__init__.py | 4 + openlane_scraper/openlane/auth.py | 484 ++++++++++++++++++++ openlane_scraper/openlane/checkpoint.py | 43 ++ openlane_scraper/openlane/client.py | 376 ++++++++++++++++ openlane_scraper/openlane/mapper.py | 424 +++++++++++++++++ openlane_scraper/openlane/runner.py | 273 +++++++++++ openlane_scraper/openlane/writer.py | 55 +++ openlane_scraper/scraper.py | 514 +++++++++++++++++++++ openlane_scraper/storage/__init__.py | 1 + openlane_scraper/storage/db.py | 576 ++++++++++++++++++++++++ openlane_scraper/storage/enums.py | 25 + openlane_scraper/storage/models.py | 82 ++++ openlane_scraper/storage/schemas.py | 87 ++++ openlane_scraper/worker/__init__.py | 3 + openlane_scraper/worker/celery_app.py | 124 +++++ openlane_scraper/worker/tasks.py | 360 +++++++++++++++ 32 files changed, 4713 insertions(+) create mode 100644 openlane_scraper/__init__.py create mode 100644 openlane_scraper/api/__init__.py create mode 100644 openlane_scraper/api/app.py create mode 100644 openlane_scraper/api/deps.py create mode 100644 openlane_scraper/api/routes/__init__.py create mode 100644 openlane_scraper/api/routes/cars.py create mode 100644 openlane_scraper/api/routes/health.py create mode 100644 openlane_scraper/api/routes/tasks.py create mode 100644 openlane_scraper/browser/__init__.py create mode 100644 openlane_scraper/browser/factory.py create mode 100644 openlane_scraper/cli.py create mode 100644 openlane_scraper/core/__init__.py create mode 100644 openlane_scraper/core/config.py create mode 100644 openlane_scraper/core/logs.py create mode 100644 openlane_scraper/core/runtime_config.py create mode 100644 openlane_scraper/core/utils.py create mode 100644 openlane_scraper/openlane/__init__.py create mode 100644 openlane_scraper/openlane/auth.py create mode 100644 openlane_scraper/openlane/checkpoint.py create mode 100644 openlane_scraper/openlane/client.py create mode 100644 openlane_scraper/openlane/mapper.py create mode 100644 openlane_scraper/openlane/runner.py create mode 100644 openlane_scraper/openlane/writer.py create mode 100644 openlane_scraper/scraper.py create mode 100644 openlane_scraper/storage/__init__.py create mode 100644 openlane_scraper/storage/db.py create mode 100644 openlane_scraper/storage/enums.py create mode 100644 openlane_scraper/storage/models.py create mode 100644 openlane_scraper/storage/schemas.py create mode 100644 openlane_scraper/worker/__init__.py create mode 100644 openlane_scraper/worker/celery_app.py create mode 100644 openlane_scraper/worker/tasks.py diff --git a/openlane_scraper/__init__.py b/openlane_scraper/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/openlane_scraper/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/openlane_scraper/api/__init__.py b/openlane_scraper/api/__init__.py new file mode 100644 index 0000000..b94a1e8 --- /dev/null +++ b/openlane_scraper/api/__init__.py @@ -0,0 +1,3 @@ +from .app import create_app + +__all__ = ["create_app"] diff --git a/openlane_scraper/api/app.py b/openlane_scraper/api/app.py new file mode 100644 index 0000000..ed66834 --- /dev/null +++ b/openlane_scraper/api/app.py @@ -0,0 +1,40 @@ +# Создание FastAPI-приложения и настройка его жизненного цикла. + +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from ..core.config import Settings +from ..storage.db import PersistenceService +from .routes import cars, health, tasks + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Жизненный цикл. + # Таблицы создаются через Alembic-миграции (сервис migrate). + yield + + +def create_app(settings: Settings | None = None) -> FastAPI: + _settings = settings or Settings() + + app = FastAPI( + title="OpenLane Scraper API", + description="REST API для управления задачами скрапинга OpenLane и просмотра данных", + version="1.0.0", + lifespan=lifespan, + ) + + app.state.settings = _settings + app.state.persistence = PersistenceService(_settings) + + app.include_router(health.router, tags=["health"]) + app.include_router(cars.router, prefix="/api/v1", tags=["cars"]) + app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"]) + + return app + + +# Экземпляр приложения для запуска через uvicorn. +app = create_app() diff --git a/openlane_scraper/api/deps.py b/openlane_scraper/api/deps.py new file mode 100644 index 0000000..86b4bc3 --- /dev/null +++ b/openlane_scraper/api/deps.py @@ -0,0 +1,9 @@ +# Вспомогательные зависимости для FastAPI-роутов. + +from fastapi import Request + +from ..storage.db import PersistenceService + + +def get_persistence(request: Request) -> PersistenceService: + return request.app.state.persistence diff --git a/openlane_scraper/api/routes/__init__.py b/openlane_scraper/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/openlane_scraper/api/routes/cars.py b/openlane_scraper/api/routes/cars.py new file mode 100644 index 0000000..7195d48 --- /dev/null +++ b/openlane_scraper/api/routes/cars.py @@ -0,0 +1,103 @@ +# Роуты для просмотра автомобилей и агрегированной статистики. + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import Car, Image +from ...storage.schemas import CarRead + +router = APIRouter() + + +@router.get("/cars") +def list_cars( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + brand: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + is_sold: bool | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + # Список автомобилей с пагинацией и фильтрами + with persistence.session_scope() as session: + query = select(Car) + + if brand: + query = query.where(Car.brand.ilike(f"%{brand}%")) + if model: + query = query.where(Car.model.ilike(f"%{model}%")) + if year_min is not None: + query = query.where(Car.year >= year_min) + if year_max is not None: + query = query.where(Car.year <= year_max) + if is_sold is not None: + query = query.where(Car.is_sold == is_sold) + + count_query = select(func.count()).select_from(query.subquery()) + total = session.execute(count_query).scalar() or 0 + + offset = (page - 1) * per_page + cars = session.execute( + query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if per_page else 0, + "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars], + } + + +@router.get("/cars/{car_id}") +def get_car( + car_id: int, + persistence: PersistenceService = Depends(get_persistence), +): + # Детальная информация об автомобиле с изображениями + with persistence.session_scope() as session: + car = session.get(Car, car_id) + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/cars/by-origin/{origin_id}") +def get_car_by_origin( + origin_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + # Поиск автомобиля по origin_id + with persistence.session_scope() as session: + car = session.execute( + select(Car).where(Car.origin_id == origin_id) + ).scalars().first() + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/stats") +def get_stats(persistence: PersistenceService = Depends(get_persistence)): + # Общая статистика по БД + with persistence.session_scope() as session: + total_cars = session.execute(select(func.count(Car.id))).scalar() or 0 + total_images = session.execute(select(func.count(Image.id))).scalar() or 0 + brands = session.execute( + select(Car.brand, func.count(Car.id)) + .group_by(Car.brand) + .order_by(func.count(Car.id).desc()) + .limit(20) + ).all() + + return { + "total_cars": total_cars, + "total_images": total_images, + "top_brands": [{"brand": b, "count": c} for b, c in brands], + } + diff --git a/openlane_scraper/api/routes/health.py b/openlane_scraper/api/routes/health.py new file mode 100644 index 0000000..65273b3 --- /dev/null +++ b/openlane_scraper/api/routes/health.py @@ -0,0 +1,30 @@ +# Роут проверки доступности сервиса и соединения с БД. + +import logging + +from fastapi import APIRouter, Depends +from sqlalchemy import text + +from ..deps import get_persistence +from ...storage.db import PersistenceService + +router = APIRouter() +logger = logging.getLogger("openlane_scraper.api.health") + + +@router.get("/health") +def health_check(persistence: PersistenceService = Depends(get_persistence)): + # Проверка API и БД + db_ok = False + try: + with persistence.session_scope() as session: + session.execute(text("SELECT 1")) + db_ok = True + except Exception: + logger.warning("Health DB check failed", exc_info=True) + + return { + "status": "ok" if db_ok else "degraded", + "service": "openlane-scraper-api", + "database": "connected" if db_ok else "unavailable", + } diff --git a/openlane_scraper/api/routes/tasks.py b/openlane_scraper/api/routes/tasks.py new file mode 100644 index 0000000..e277ded --- /dev/null +++ b/openlane_scraper/api/routes/tasks.py @@ -0,0 +1,96 @@ +# Роуты запуска задач синхронизации и просмотра истории sync-runs. + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from sqlalchemy import select, func + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import SyncRun +from ...worker.celery_app import celery_app +from ...worker.tasks import sync_listing_task + +router = APIRouter() + + +class SyncListingRequest(BaseModel): + lane: str = "openlane_marketplace" + limit: int | None = None + only_new: bool | None = None + max_pages: int | None = None + concurrency: int | None = None + + +@router.post("/tasks/sync-listing") +def start_sync_listing( + body: SyncListingRequest, +): + result = sync_listing_task.apply_async( + kwargs={ + "lane": body.lane, + "limit": body.limit, + "only_new": body.only_new, + "max_pages": body.max_pages, + "concurrency": body.concurrency, + }, + queue="scraping", + ) + + return { + "task_id": result.id, + "status": "queued", + } + + +@router.get("/tasks/{task_id}") +def get_task_status(task_id: str): + result = celery_app.AsyncResult(task_id) + + payload: dict = { + "task_id": task_id, + "state": result.state, + } + + if result.successful(): + payload["result"] = result.result + elif result.failed(): + payload["error"] = str(result.result) + elif result.info is not None: + payload["meta"] = result.info + + return payload + + +@router.get("/sync-runs") +def list_sync_runs( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + persistence: PersistenceService = Depends(get_persistence), +): + with persistence.session_scope() as session: + total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 + offset = (page - 1) * per_page + runs = session.execute( + select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": r.id, + "started_at": r.started_at.isoformat() if r.started_at else None, + "finished_at": r.finished_at.isoformat() if r.finished_at else None, + "status": r.status, + "lane": r.lane, + "ids_fetched": r.ids_fetched, + "cars_upserted": r.cars_upserted, + "cars_failed": r.cars_failed, + "images_upserted": r.images_upserted, + "error_summary": r.error_summary, + } + for r in runs + ], + } diff --git a/openlane_scraper/browser/__init__.py b/openlane_scraper/browser/__init__.py new file mode 100644 index 0000000..9def84b --- /dev/null +++ b/openlane_scraper/browser/__init__.py @@ -0,0 +1,3 @@ +from .factory import BrowserFactory + +__all__ = ["BrowserFactory"] diff --git a/openlane_scraper/browser/factory.py b/openlane_scraper/browser/factory.py new file mode 100644 index 0000000..1a4954b --- /dev/null +++ b/openlane_scraper/browser/factory.py @@ -0,0 +1,226 @@ +import json +import logging +import random + +from playwright.sync_api import Browser, BrowserContext, Playwright + +try: + from playwright_stealth import stealth_sync +except ImportError: + stealth_sync = None + +from ..core.config import Settings + +logger = logging.getLogger("openlane_scraper.browser") + + +def _build_init_script() -> str: + # Патч признаков автоматизации. + hardware_concurrency = random.choice([4, 8, 12, 16]) + device_memory = random.choice([4, 8, 16]) + languages = ["en-US", "en"] + + return f""" +(() => {{ + const define = (obj, prop, value) => {{ + try {{ + Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); + }} catch (e) {{}} + }}; + + define(navigator, 'webdriver', undefined); + define(navigator, 'platform', 'Win32'); + define(navigator, 'vendor', 'Google Inc.'); + define(navigator, 'language', '{languages[0]}'); + define(navigator, 'languages', {json.dumps(languages)}); + define(navigator, 'hardwareConcurrency', {hardware_concurrency}); + define(navigator, 'deviceMemory', {device_memory}); + define(navigator, 'maxTouchPoints', 0); + + if (!window.chrome) {{ + Object.defineProperty(window, 'chrome', {{ + value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, + configurable: true + }}); + }} + + const originalQuery = navigator.permissions && navigator.permissions.query; + if (originalQuery) {{ + navigator.permissions.query = (params) => ( + params && params.name === 'notifications' + ? Promise.resolve({{ state: Notification.permission }}) + : originalQuery(params) + ); + }} + + const originalGetParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(parameter) {{ + if (parameter === 37445) return 'Intel Inc.'; + if (parameter === 37446) return 'Intel Iris OpenGL Engine'; + return originalGetParameter.call(this, parameter); + }}; +}})(); +""" + + +class BrowserFactory: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + + def _resolve_engine(self) -> str: + # Выбор движка браузера. + engine = self.settings.browser_engine.strip().lower() + if engine == "auto": + # В headless-режиме Chromium со stealth-скриптами + # значительно стабильнее Firefox по anti-bot. + return "chromium" + if engine in ("firefox", "chromium"): + return engine + logger.warning("Unknown OPENLANE_BROWSER_ENGINE=%r, falling back to auto", engine) + return "chromium" + + def create_browser(self, playwright: Playwright) -> Browser: + engine = self._resolve_engine() + proxy_dict = self.settings.proxy.to_playwright_dict() + logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine) + + if engine == "firefox": + launch_kwargs: dict = {"headless": self.settings.headless} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + # Параметры Firefox для headless-режима. + launch_kwargs["firefox_user_prefs"] = { + "dom.webdriver.enabled": False, + "useAutomationExtension": False, + # Базовые оптимизации для VPS. + "media.autoplay.default": 5, + "media.volume_scale": "0.0", + "media.audio.playback.standalone": False, + "dom.ipc.processCount": 1, + "dom.ipc.plugins.enabled": False, + "browser.cache.disk.enable": False, + "browser.cache.memory.enable": True, + "browser.cache.memory.max_entry_size": 8192, + "network.prefetch-next": False, + "network.dns.disablePrefetch": True, + "permissions.default.image": 2, + "javascript.options.mem.gc_incremental_mark_slice_ms": 20, + } + logger.info("Launching Firefox (headless=%s)", self.settings.headless) + return playwright.firefox.launch(**launch_kwargs) + + # Путь запуска Chromium. + args = [ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-features=IsolateOrigins,site-per-process", + ] + if self.settings.headless: + args.append("--headless=new") + pw_headless = False + logger.info("Using Chromium new-headless mode (--headless=new)") + else: + pw_headless = False + + launch_kwargs = {"headless": pw_headless, "args": args} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + try: + logger.info("Trying to launch real Chrome channel") + return playwright.chromium.launch(channel="chrome", **launch_kwargs) + except Exception: + logger.warning("Chrome channel launch failed, falling back to Chromium") + return playwright.chromium.launch(**launch_kwargs) + + def create_context(self, browser: Browser, storage_state_path: str | None = None) -> BrowserContext: + viewport = random.choice(self.settings.fingerprint.viewport_presets) + timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) + color_scheme = random.choice(["light", "dark"]) + + is_firefox = browser.browser_type.name == "firefox" + + ctx_kwargs: dict = { + "viewport": viewport, + "screen": viewport, + "locale": self.settings.fingerprint.locale, + "timezone_id": timezone_id, + "color_scheme": color_scheme, + "java_script_enabled": True, + "ignore_https_errors": False, + } + + if is_firefox: + # Заголовки и user-agent для Firefox. + ctx_kwargs["user_agent"] = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + } + else: + ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent + ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25]) + ctx_kwargs["is_mobile"] = False + ctx_kwargs["has_touch"] = False + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.9", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', + } + + if storage_state_path: + ctx_kwargs["storage_state"] = storage_state_path + + context = browser.new_context(**ctx_kwargs) + context.set_default_timeout(self.settings.default_timeout_ms) + context.set_default_navigation_timeout(self.settings.default_timeout_ms) + + if not is_firefox: + # Патчи маскировки для Chromium. + context.add_init_script(_build_init_script()) + if stealth_sync: + context.on("page", lambda page: stealth_sync(page)) + logger.debug("playwright-stealth attached to context") + + return context + + @staticmethod + def enable_resource_blocking(page) -> None: + # Блокируем всё кроме document и XHR/fetch — минимальный след. + BLOCKED_TYPES = {"image", "stylesheet", "font", "media", "websocket", "eventsource", "manifest", "other"} + BLOCKED_URL_PATTERNS = ( + "google-analytics", "googletagmanager", "facebook.net", + "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2", + "analytics", "tracking", "adservice", + # OpenLane-трекеры из сетевого лога. + "bugsnag", "sessions.bugsnag", + "intercom", "widget.intercom", + "pusher", "sockjs", + "segment", "cdn.segment", + "ap3.com", "ap3c.com", + "onetrust", "optanon", "cookielaw", + "fullstory", "sentry", + ) + + def _handle_route(route): + req = route.request + if req.resource_type in BLOCKED_TYPES: + route.abort() + return + url = req.url.lower() + if any(pat in url for pat in BLOCKED_URL_PATTERNS): + route.abort() + return + route.continue_() + + page.route("**/*", _handle_route) diff --git a/openlane_scraper/cli.py b/openlane_scraper/cli.py new file mode 100644 index 0000000..d24897a --- /dev/null +++ b/openlane_scraper/cli.py @@ -0,0 +1,92 @@ +import argparse +from pathlib import Path + +from .core.config import Settings +from .core.utils import save_to_json +from .openlane import OpenLaneScrapeRunner +from .scraper import OpenLaneScraper + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="OpenLane scraper CLI") + parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") + parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("init-db", help="Create DB tables") + + sync_parser = subparsers.add_parser("sync-listing", help="Sync OpenLane marketplace → DB") + sync_parser.add_argument("--lane", default="openlane_marketplace") + sync_parser.add_argument("--limit", type=int, default=None) + sync_parser.add_argument("--only-new", choices=["true", "false"], default=None) + sync_parser.add_argument("--max-pages", type=int, default=None) + sync_parser.add_argument("--concurrency", type=int, default=None) + sync_parser.add_argument("--output", default=str(Path("artifacts/json/openlane_sync_listing.json"))) + + scrape_parser = subparsers.add_parser("scrape-openlane", help="Scrape OpenLane marketplace to JSONL files") + scrape_parser.add_argument("--max-pages", type=int, default=None) + scrape_parser.add_argument("--concurrency", type=int, default=None) + scrape_parser.add_argument("--resume", action="store_true") + scrape_parser.add_argument("--checkpoint-every-pages", type=int, default=None) + + subparsers.add_parser("openlane-login", help="Open interactive OpenLane login and persist browser storage state") + + return parser + + +def main() -> None: + parser = build_parser() + args = parser.parse_args() + + runtime_settings: Settings | None = None + if args.headless is not None or args.debug: + runtime_settings = Settings() + if args.headless is not None: + runtime_settings.headless = args.headless == "true" + if args.debug: + runtime_settings.log_level = "DEBUG" + + if args.command == "openlane-login": + runner = OpenLaneScrapeRunner(runtime_settings) + storage_state_path = runner.interactive_login() + print(f"OpenLane storage state saved to {Path(storage_state_path).resolve()}") + return + + if args.command == "scrape-openlane": + runner = OpenLaneScrapeRunner(runtime_settings) + result = runner.run( + max_pages=args.max_pages, + concurrency=args.concurrency, + resume=args.resume, + checkpoint_every_pages=args.checkpoint_every_pages, + ) + print(f"OpenLane JSONL saved to {Path(result.jsonl_path).resolve()}") + print(f"OpenLane aggregated JSON saved to {Path(result.aggregated_path).resolve()}") + print(f"OpenLane checkpoint saved to {Path(result.checkpoint_path).resolve()}") + print(f"OpenLane storage state saved to {Path(result.storage_state_path).resolve()}") + print( + f"OpenLane completed pages={len(result.completed_pages)} failed pages={len(result.failed_pages)} total_records={result.total_records} elapsed={result.elapsed_seconds:.2f}s" + ) + return + + with OpenLaneScraper(runtime_settings) as scraper: + if args.command == "init-db": + data = scraper.init_db() + print(f"DB initialized: {data}") + return + elif args.command == "sync-listing": + only_new = None if args.only_new is None else args.only_new == "true" + data = scraper.sync_listing( + lane=args.lane, + limit=args.limit, + only_new=only_new, + max_pages=args.max_pages, + concurrency=args.concurrency, + ) + + save_to_json(data, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + + +if __name__ == "__main__": + main() diff --git a/openlane_scraper/core/__init__.py b/openlane_scraper/core/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/openlane_scraper/core/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/openlane_scraper/core/config.py b/openlane_scraper/core/config.py new file mode 100644 index 0000000..fdac325 --- /dev/null +++ b/openlane_scraper/core/config.py @@ -0,0 +1,194 @@ +import os +from dataclasses import dataclass, field +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +# Хелперы для чтения env-переменных с приведением типов + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + return value if value is not None else default + + +def _env_optional_str(name: str) -> str | None: + value = os.getenv(name) + if value is None: + return None + value = value.strip() + return value or None + + +def _env_path_str(name: str) -> str | None: + value = _env_optional_str(name) + if value is None: + return None + return str(Path(value).expanduser()) + + +def _env_bool(name: str, default: bool) -> bool: + fallback = "true" if default else "false" + return _env_str(name, fallback).strip().lower() in TRUE_VALUES + + +def _env_int(name: str, default: int) -> int: + return int(_env_str(name, str(default)).strip()) + + +def _env_float(name: str, default: float) -> float: + return float(_env_str(name, str(default)).strip()) + + +# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +# --- Конфиг OpenLane (auth, paging, throttle, retry, checkpoint/output) --- + +@dataclass(slots=True) +class OpenLaneConfig: + sign_in_url: str = _env_str("OPENLANE_SIGN_IN_URL", "https://app.openlane.com/sign_in") + search_url: str = _env_str("OPENLANE_SEARCH_URL", "https://app.openlane.com/search?tab=marketplace") + api_search_url: str = _env_str("OPENLANE_API_SEARCH_URL", "https://app.openlane.com/api/v4/search") + username: str = _env_str("OPENLANE_USERNAME", "") + password: str = _env_str("OPENLANE_PASSWORD", "") + source_tab: str = _env_str("OPENLANE_SOURCE_TAB", "marketplace") + sale_types: str = _env_str("OPENLANE_SALE_TYPES", "marketplace") + page_size: int = _env_int("OPENLANE_PAGE_SIZE", 30) + max_pages: int = _env_int("OPENLANE_MAX_PAGES", 512) + concurrency: int = _env_int("OPENLANE_CONCURRENCY", 4) + throttle_min_seconds: float = _env_float("OPENLANE_THROTTLE_MIN_SECONDS", 0.3) + throttle_max_seconds: float = _env_float("OPENLANE_THROTTLE_MAX_SECONDS", 0.8) + retry_schedule_seconds: tuple[float, ...] = tuple( + float(part.strip()) + for part in _env_str("OPENLANE_RETRY_SCHEDULE_SECONDS", "2,5,10").split(",") + if part.strip() + ) or (2.0, 5.0, 10.0) + checkpoint_every_pages: int = _env_int("OPENLANE_CHECKPOINT_EVERY_PAGES", 10) + output_dir: str = _env_str("OPENLANE_OUTPUT_DIR", "artifacts/openlane") + jsonl_output: str = _env_str("OPENLANE_JSONL_OUTPUT", "artifacts/openlane/openlane_search.jsonl") + aggregated_output: str = _env_str("OPENLANE_AGGREGATED_OUTPUT", "artifacts/openlane/openlane_search_aggregated.json") + checkpoint_file: str = _env_str("OPENLANE_CHECKPOINT_FILE", "artifacts/openlane/openlane_checkpoint.json") + storage_state_file: str = _env_str("OPENLANE_STORAGE_STATE_FILE", "artifacts/openlane/openlane_storage_state.json") + persist_storage_state: bool = _env_bool("OPENLANE_PERSIST_STORAGE_STATE", True) + progress_log_every_pages: int = _env_int("OPENLANE_PROGRESS_LOG_EVERY_PAGES", 10) + request_timeout_ms: int = _env_int("OPENLANE_REQUEST_TIMEOUT_MS", 45000) + max_cars_limit: int = _env_int("OPENLANE_MAX_CARS_LIMIT", 20) + refresh_token: str = _env_str("OPENLANE_REFRESH_TOKEN", "") + okta_client_id: str = _env_str("OPENLANE_OKTA_CLIENT_ID", "") + okta_token_endpoint: str = _env_str( + "OPENLANE_OKTA_TOKEN_ENDPOINT", + "https://okta.iam.karglobal.com/oauth2/default/v1/token", + ) + okta_redirect_uri: str = _env_str("OPENLANE_OKTA_REDIRECT_URI", "https://app.openlane.com/sign_in") + okta_timeout_seconds: int = _env_int("OPENLANE_OKTA_TIMEOUT_SECONDS", 15) + + +# --- Конфиг PostgreSQL (URL, пул соединений, pool_recycle) --- + +@dataclass(slots=True) +class DatabaseConfig: + url: str = _env_str("OPENLANE_DATABASE_URL", "postgresql+psycopg2://openlane:openlane@localhost:5432/openlane_scraper") + echo: bool = _env_bool("OPENLANE_DATABASE_ECHO", False) + pool_size: int = _env_int("OPENLANE_DATABASE_POOL_SIZE", 5) + max_overflow: int = _env_int("OPENLANE_DATABASE_MAX_OVERFLOW", 10) + pool_recycle_seconds: int = _env_int("OPENLANE_DATABASE_POOL_RECYCLE_SECONDS", 1800) + auto_create_tables: bool = _env_bool("OPENLANE_DATABASE_AUTO_CREATE_TABLES", False) + + +# --- Конфиг Redis (URL для Celery broker) --- + +@dataclass(slots=True) +class RedisConfig: + url: str = _env_str("OPENLANE_REDIS_URL", "redis://localhost:6379/0") + socket_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) + socket_connect_timeout_seconds: float = _env_float("OPENLANE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) + health_check_interval_seconds: int = _env_int("OPENLANE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) + + +# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = _env_str("CELERY_BROKER_URL", "") + result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") + task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) + task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) + worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) + worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) + broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) + beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) + beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None + + +# --- Конфиг прокси (server, username, password) --- + +@dataclass(slots=True) +class ProxyConfig: + server: str | None = _env_optional_str("OPENLANE_PROXY_SERVER") + username: str | None = _env_optional_str("OPENLANE_PROXY_USERNAME") + password: str | None = _env_optional_str("OPENLANE_PROXY_PASSWORD") + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + +# Главный объект настроек: собирает все блоки конфигурации + +@dataclass(slots=True) +class Settings: + default_timeout_ms: int = _env_int("OPENLANE_TIMEOUT_MS", 45000) + headless: bool = _env_bool("OPENLANE_HEADLESS", True) + browser_engine: str = _env_str("OPENLANE_BROWSER_ENGINE", "auto") + log_level: str = _env_str("OPENLANE_LOG_LEVEL", "INFO") + log_file: str | None = _env_optional_str("OPENLANE_LOG_FILE") + enable_trace_id_logs: bool = _env_bool("OPENLANE_ENABLE_TRACE_ID_LOGS", True) + runtime_config_file: str | None = _env_path_str("OPENLANE_RUNTIME_CONFIG_FILE") + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + openlane: OpenLaneConfig = field(default_factory=OpenLaneConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) + + +# Глобальный синглтон — используется по умолчанию во всех модулях. +settings = Settings() diff --git a/openlane_scraper/core/logs.py b/openlane_scraper/core/logs.py new file mode 100644 index 0000000..f1c324b --- /dev/null +++ b/openlane_scraper/core/logs.py @@ -0,0 +1,39 @@ +import logging +import sys +from contextvars import ContextVar + +# Храним trace_id текущего потока/корутины. +TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") + + +class TraceIdFilter(logging.Filter): + # Добавляет trace_id в каждую запись лога для сквозной трассировки. + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = TRACE_ID.get() + return True + + +def set_trace_id(trace_id: str) -> None: + TRACE_ID.set(trace_id) + + +def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + # stderr — Docker и Celery prefork корректно его подхватывают. + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)] + if log_file: + handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + trace_filter = TraceIdFilter() + for handler in handlers: + handler.addFilter(trace_filter) + handler.setLevel(getattr(logging, level.upper(), logging.INFO)) + root = logging.getLogger() + root.setLevel(getattr(logging, level.upper(), logging.INFO)) + # Убираем старые хендлеры, чтобы не дублировать после fork. + root.handlers.clear() + for handler in handlers: + root.addHandler(handler) + fmt = logging.Formatter( + "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s" + ) + for handler in root.handlers: + handler.setFormatter(fmt) diff --git a/openlane_scraper/core/runtime_config.py b/openlane_scraper/core/runtime_config.py new file mode 100644 index 0000000..afe6f84 --- /dev/null +++ b/openlane_scraper/core/runtime_config.py @@ -0,0 +1,379 @@ +"""Runtime config: загрузка и применение runtime_config.json. + +Позволяет менять параметры sync и фильтры машин без перезапуска сервиса. +Файл перечитывается при каждом запуске sync_listing. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger("openlane_scraper.core.runtime_config") + + +@dataclass(slots=True) +class SyncConfig: + name: str | None = None + ids_initial_size: int | None = None + ids_next_size: int | None = None + ids_max_pages: int | None = None + condition_check_enabled: bool | None = None + lane: str = "openlane_marketplace" + only_new: bool = False + limit: int | None = None + + +@dataclass(slots=True) +class ListingConfig: + make: str | None = None + model: str | None = None + + +@dataclass(slots=True) +class RangeFilter: + min: int | None = None + max: int | None = None + + +@dataclass(slots=True) +class FlagFilters: + damaged_only: bool | None = None + run_and_drive: bool | None = None + + +@dataclass(slots=True) +class FieldFilters: + brands: list[str] = field(default_factory=list) + models: list[str] = field(default_factory=list) + years: list[int] = field(default_factory=list) + body_types: list[str] = field(default_factory=list) + colors: list[str] = field(default_factory=list) + drives: list[str] = field(default_factory=list) + gearboxes: list[str] = field(default_factory=list) + locations: list[str] = field(default_factory=list) + + +@dataclass(slots=True) +class Filters: + price: RangeFilter = field(default_factory=RangeFilter) + mileage: RangeFilter = field(default_factory=RangeFilter) + flags: FlagFilters = field(default_factory=FlagFilters) + + include: FieldFilters = field(default_factory=FieldFilters) + exclude: FieldFilters = field(default_factory=FieldFilters) + + # Legacy flat fields (backward compatibility). + brands: list[str] = field(default_factory=list) + models: list[str] = field(default_factory=list) + years: list[int] = field(default_factory=list) + body_types: list[str] = field(default_factory=list) + colors: list[str] = field(default_factory=list) + drives: list[str] = field(default_factory=list) + gearboxes: list[str] = field(default_factory=list) + locations: list[str] = field(default_factory=list) + + exclude_brands: list[str] = field(default_factory=list) + exclude_models: list[str] = field(default_factory=list) + exclude_years: list[int] = field(default_factory=list) + exclude_body_types: list[str] = field(default_factory=list) + exclude_colors: list[str] = field(default_factory=list) + exclude_drives: list[str] = field(default_factory=list) + exclude_gearboxes: list[str] = field(default_factory=list) + exclude_locations: list[str] = field(default_factory=list) + + +@dataclass(slots=True) +class RuntimeConfig: + sync: SyncConfig = field(default_factory=SyncConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + filters: Filters = field(default_factory=Filters) + + +def load_runtime_config(path: str | Path | None) -> RuntimeConfig: + """Загрузить runtime_config.json. Если файл отсутствует — вернуть дефолты.""" + if not path: + return RuntimeConfig() + + p = Path(path) + if not p.exists(): + logger.debug("runtime_config not found at %s — using defaults", p) + return RuntimeConfig() + + try: + raw = json.loads(p.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + logger.warning("Failed to parse runtime_config %s: %s — using defaults", p, exc) + return RuntimeConfig() + + cfg = RuntimeConfig() + + sync_raw = raw.get("sync") or {} + if isinstance(sync_raw, dict): + cfg.sync.name = _opt_str(sync_raw.get("name")) + cfg.sync.ids_initial_size = _opt_int(sync_raw.get("ids_initial_size")) + cfg.sync.ids_next_size = _opt_int(sync_raw.get("ids_next_size")) + cfg.sync.ids_max_pages = _opt_int(sync_raw.get("ids_max_pages")) + cfg.sync.condition_check_enabled = _opt_bool(sync_raw.get("condition_check_enabled")) + cfg.sync.lane = _opt_str(sync_raw.get("lane")) or cfg.sync.lane + only_new = _opt_bool(sync_raw.get("only_new")) + if only_new is not None: + cfg.sync.only_new = only_new + cfg.sync.limit = _opt_int(sync_raw.get("limit")) + + listing_raw = raw.get("listing") or {} + if isinstance(listing_raw, dict): + cfg.listing.make = _opt_str(listing_raw.get("make")) + cfg.listing.model = _opt_str(listing_raw.get("model")) + + filters_raw = raw.get("filters") or {} + if isinstance(filters_raw, dict): + cfg.filters.price = _parse_range(filters_raw.get("price")) + cfg.filters.mileage = _parse_range(filters_raw.get("mileage")) + cfg.filters.flags = _parse_flags(filters_raw.get("flags")) + + include_payload = filters_raw.get("include") if isinstance(filters_raw.get("include"), dict) else filters_raw + exclude_payload = ( + filters_raw.get("exclude") + if isinstance(filters_raw.get("exclude"), dict) + else { + "brands": filters_raw.get("exclude_brands"), + "models": filters_raw.get("exclude_models"), + "years": filters_raw.get("exclude_years"), + "body_types": filters_raw.get("exclude_body_types"), + "colors": filters_raw.get("exclude_colors"), + "drives": filters_raw.get("exclude_drives"), + "gearboxes": filters_raw.get("exclude_gearboxes"), + "locations": filters_raw.get("exclude_locations"), + } + ) + + cfg.filters.include = _parse_fields(include_payload) + cfg.filters.exclude = _parse_fields(exclude_payload) + + # Backward-compatible flat aliases. + cfg.filters.brands = list(cfg.filters.include.brands) + cfg.filters.models = list(cfg.filters.include.models) + cfg.filters.years = list(cfg.filters.include.years) + cfg.filters.body_types = list(cfg.filters.include.body_types) + cfg.filters.colors = list(cfg.filters.include.colors) + cfg.filters.drives = list(cfg.filters.include.drives) + cfg.filters.gearboxes = list(cfg.filters.include.gearboxes) + cfg.filters.locations = list(cfg.filters.include.locations) + + cfg.filters.exclude_brands = list(cfg.filters.exclude.brands) + cfg.filters.exclude_models = list(cfg.filters.exclude.models) + cfg.filters.exclude_years = list(cfg.filters.exclude.years) + cfg.filters.exclude_body_types = list(cfg.filters.exclude.body_types) + cfg.filters.exclude_colors = list(cfg.filters.exclude.colors) + cfg.filters.exclude_drives = list(cfg.filters.exclude.drives) + cfg.filters.exclude_gearboxes = list(cfg.filters.exclude.gearboxes) + cfg.filters.exclude_locations = list(cfg.filters.exclude.locations) + + logger.info( + "Loaded runtime_config from %s: lane=%s, limit=%s", + p, + cfg.sync.lane, + cfg.sync.limit, + ) + return cfg + + +def apply_filters(records: list[dict], filters: Filters) -> list[dict]: + """Применить runtime-фильтры к списку сырых записей из API.""" + if not records: + return records + + include = getattr(filters, "include", None) + exclude = getattr(filters, "exclude", None) + + include_brands = tuple(getattr(filters, "brands", ()) or getattr(include, "brands", ())) + include_models = tuple(getattr(filters, "models", ()) or getattr(include, "models", ())) + include_years = tuple(getattr(filters, "years", ()) or getattr(include, "years", ())) + include_body_types = tuple(getattr(filters, "body_types", ()) or getattr(include, "body_types", ())) + + exclude_brands = tuple(getattr(filters, "exclude_brands", ()) or getattr(exclude, "brands", ())) + exclude_models = tuple(getattr(filters, "exclude_models", ()) or getattr(exclude, "models", ())) + exclude_years = tuple(getattr(filters, "exclude_years", ()) or getattr(exclude, "years", ())) + exclude_body_types = tuple(getattr(filters, "exclude_body_types", ()) or getattr(exclude, "body_types", ())) + + price_cfg = getattr(filters, "price", None) + mileage_cfg = getattr(filters, "mileage", None) + flags_cfg = getattr(filters, "flags", None) + + price_min = getattr(price_cfg, "min", None) + price_max = getattr(price_cfg, "max", None) + mileage_min = getattr(mileage_cfg, "min", None) + mileage_max = getattr(mileage_cfg, "max", None) + damaged_only = getattr(flags_cfg, "damaged_only", None) + run_and_drive = getattr(flags_cfg, "run_and_drive", None) + + result = records + + if include_brands: + include_set = {s.casefold() for s in include_brands} + result = [r for r in result if _get_brand(r).casefold() in include_set] + if exclude_brands: + exclude_set = {s.casefold() for s in exclude_brands} + result = [r for r in result if _get_brand(r).casefold() not in exclude_set] + + if include_models: + include_set = {s.casefold() for s in include_models} + result = [r for r in result if _get_model(r).casefold() in include_set] + if exclude_models: + exclude_set = {s.casefold() for s in exclude_models} + result = [r for r in result if _get_model(r).casefold() not in exclude_set] + + if include_years: + years_set = set(include_years) + result = [r for r in result if _get_year(r) in years_set] + if exclude_years: + years_set = set(exclude_years) + result = [r for r in result if _get_year(r) not in years_set] + + if include_body_types: + include_set = {s.casefold() for s in include_body_types} + result = [r for r in result if _get_body_type(r).casefold() in include_set] + if exclude_body_types: + exclude_set = {s.casefold() for s in exclude_body_types} + result = [r for r in result if _get_body_type(r).casefold() not in exclude_set] + + if price_min is not None or price_max is not None: + filtered: list[dict] = [] + for r in result: + price = _get_price(r) + if price is None: + filtered.append(r) + continue + if price_min is not None and price < price_min: + continue + if price_max is not None and price > price_max: + continue + filtered.append(r) + result = filtered + + if mileage_min is not None or mileage_max is not None: + filtered = [] + for r in result: + miles = _get_mileage(r) + if miles is None: + filtered.append(r) + continue + if mileage_min is not None and miles < mileage_min: + continue + if mileage_max is not None and miles > mileage_max: + continue + filtered.append(r) + result = filtered + + if damaged_only is not None: + result = [r for r in result if bool(r.get("is_damaged")) is damaged_only] + if run_and_drive is not None: + result = [r for r in result if bool(r.get("run_and_drive")) is run_and_drive] + + return result + + +def _parse_range(payload: Any) -> RangeFilter: + payload = payload if isinstance(payload, dict) else {} + return RangeFilter(min=_opt_int(payload.get("min")), max=_opt_int(payload.get("max"))) + + +def _parse_flags(payload: Any) -> FlagFilters: + payload = payload if isinstance(payload, dict) else {} + return FlagFilters( + damaged_only=_opt_bool(payload.get("damaged_only")), + run_and_drive=_opt_bool(payload.get("run_and_drive")), + ) + + +def _parse_fields(payload: Any) -> FieldFilters: + payload = payload if isinstance(payload, dict) else {} + return FieldFilters( + brands=_str_list(payload.get("brands")), + models=_str_list(payload.get("models")), + years=_int_list(payload.get("years")), + body_types=_str_list(payload.get("body_types")), + colors=_str_list(payload.get("colors")), + drives=_str_list(payload.get("drives")), + gearboxes=_str_list(payload.get("gearboxes")), + locations=_str_list(payload.get("locations")), + ) + + +def _opt_int(value: Any) -> int | None: + if value is None: + return None + try: + return int(value) + except (ValueError, TypeError): + return None + + +def _opt_bool(value: Any) -> bool | None: + if value is None: + return None + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().casefold() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return None + + +def _opt_str(value: Any) -> str | None: + if value is None: + return None + s = str(value).strip() + return s or None + + +def _str_list(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + return [str(v).strip() for v in value if str(v).strip()] + + +def _int_list(value: Any) -> list[int]: + if not isinstance(value, list): + return [] + result: list[int] = [] + for v in value: + iv = _opt_int(v) + if iv is not None: + result.append(iv) + return result + + +def _get_brand(record: dict[str, Any]) -> str: + return str(record.get("make") or record.get("brand") or "") + + +def _get_model(record: dict[str, Any]) -> str: + return str(record.get("model") or "") + + +def _get_body_type(record: dict[str, Any]) -> str: + return str(record.get("body_type") or record.get("body_style") or record.get("vehicle_type") or "") + + +def _get_year(record: dict[str, Any]) -> int | None: + return _opt_int(record.get("year")) + + +def _get_price(record: dict[str, Any]) -> int | None: + for key in ("current_high_bid", "buy_now_price", "price", "current_bid", "sale_price"): + value = _opt_int(record.get(key)) + if value is not None: + return value + return None + + +def _get_mileage(record: dict[str, Any]) -> int | None: + return _opt_int(record.get("odometer") or record.get("mileage")) diff --git a/openlane_scraper/core/utils.py b/openlane_scraper/core/utils.py new file mode 100644 index 0000000..6260d6f --- /dev/null +++ b/openlane_scraper/core/utils.py @@ -0,0 +1,66 @@ +import json +import re +from pathlib import Path +from typing import Any, Callable, Iterable + + +def save_to_json(data: Any, filename: str | Path) -> None: + path = Path(filename) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def first_non_empty(values: Iterable[Any]) -> Any | None: + for value in values: + if value not in (None, "", [], {}, ()): + return value + return None + + +def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: + # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве. + found = [] + if _depth >= max_depth: + return found + if isinstance(obj, dict): + for key, value in obj.items(): + if key.lower() in target_keys: + found.append(value) + found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) + elif isinstance(obj, list): + for item in obj: + found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) + return found + + +def deep_find_all_keys( + payloads: list, + field_map: dict[str, set[str]], + max_depth: int = 64, +) -> dict[str, list]: + """Извлекает все нужные поля за один проход по JSON.""" + # Готовим обратную карту: нормализованный ключ -> имя поля. + reverse: dict[str, str] = {} + for field_name, keys in field_map.items(): + for k in keys: + reverse[k.lower()] = field_name + + result: dict[str, list] = {f: [] for f in field_map} + + def _recurse(obj: Any, depth: int) -> None: + if depth >= max_depth: + return + if isinstance(obj, dict): + for k, v in obj.items(): + field = reverse.get(k.lower()) + if field is not None: + result[field].append(v) + _recurse(v, depth + 1) + elif isinstance(obj, list): + for item in obj: + _recurse(item, depth + 1) + + for payload in payloads: + _recurse(payload, 0) + + return result diff --git a/openlane_scraper/openlane/__init__.py b/openlane_scraper/openlane/__init__.py new file mode 100644 index 0000000..b2597fc --- /dev/null +++ b/openlane_scraper/openlane/__init__.py @@ -0,0 +1,4 @@ +from .mapper import map_openlane_record, map_openlane_records +from .runner import OpenLaneScrapeRunner, OpenLaneScrapeResult + +__all__ = ["OpenLaneScrapeRunner", "OpenLaneScrapeResult", "map_openlane_record", "map_openlane_records"] diff --git a/openlane_scraper/openlane/auth.py b/openlane_scraper/openlane/auth.py new file mode 100644 index 0000000..3526c28 --- /dev/null +++ b/openlane_scraper/openlane/auth.py @@ -0,0 +1,484 @@ +from __future__ import annotations + +import base64 +import json +import logging +import os +import re +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +from playwright.sync_api import BrowserContext, Page, TimeoutError as PlaywrightTimeoutError + +from ..browser.factory import BrowserFactory +from ..core.config import OpenLaneConfig + +logger = logging.getLogger("openlane_scraper.openlane.auth") + +# Порог предупреждения об истечении refresh_token (дни). +_TOKEN_REFRESH_THRESHOLD_DAYS = 3 +_ACCESS_TOKEN_MIN_TTL_SECONDS = 300 + + +class OpenLaneAuthError(RuntimeError): + pass + + +class OpenLaneAuthenticator: + def __init__(self, config: OpenLaneConfig) -> None: + self.config = config + + def validate_credentials(self) -> None: + if not self.config.username or not self.config.password: + raise OpenLaneAuthError( + "OPENLANE_USERNAME and OPENLANE_PASSWORD must be set in the environment" + ) + + def interactive_login_and_persist(self, context: BrowserContext) -> str: + page = context.new_page() + page.set_default_timeout(self.config.request_timeout_ms) + logger.info("OpenLane interactive login: open %s and complete sign-in manually", self.config.sign_in_url) + page.goto(self.config.sign_in_url, wait_until="domcontentloaded") + self._wait_for_authenticated_session(page) + self._persist_storage_state(context) + logger.info("OpenLane interactive login successful") + return str(Path(self.config.storage_state_file)) + + def _inject_refresh_token(self, context: BrowserContext) -> None: + # Инъекция refresh_token cookie из env. + token = self.config.refresh_token + if not token: + return + + # Проверка: токен уже истёк? + days = self._token_days_remaining(token) + if days is not None and days <= 0: + logger.warning( + "OPENLANE_REFRESH_TOKEN is EXPIRED (%.1f days ago). " + "Will attempt login with username/password.", + abs(days), + ) + # Очищаем токен — fallback на login() + self.config.refresh_token = "" + return + + if days is not None: + logger.info("refresh_token: %.1f days remaining (expires %s)", + days, datetime.fromtimestamp( + self._decode_jwt_exp(token), tz=timezone.utc # type: ignore[arg-type] + ).strftime("%Y-%m-%d %H:%M UTC")) + if days < _TOKEN_REFRESH_THRESHOLD_DAYS: + logger.warning( + "refresh_token expires in %.1f days! " + "Will try to re-login after auth to get a fresh one.", + days, + ) + + context.add_cookies([ + { + "name": "refresh_token", + "value": token, + "domain": ".openlane.com", + "path": "/", + "httpOnly": True, + "secure": True, + "sameSite": "None", + } + ]) + logger.info("Injected OPENLANE_REFRESH_TOKEN cookie into browser context") + + # ------------------------------------------------------------------ + # Инспекция и ротация токенов + # ------------------------------------------------------------------ + + @staticmethod + def _decode_jwt_exp(token: str) -> int | None: + # Декодирование JWT exp (без проверки подписи). + try: + parts = token.split(".") + if len(parts) < 2: + return None + payload_b64 = parts[1] + # Дополнение base64 padding. + payload_b64 += "=" * (-len(payload_b64) % 4) + payload = json.loads(base64.urlsafe_b64decode(payload_b64)) + exp = payload.get("exp") + return int(exp) if exp is not None else None + except Exception: + return None + + @staticmethod + def _decode_jwt_payload(token: str) -> dict[str, Any]: + # Декодирование JWT payload (без проверки подписи). + try: + parts = token.split(".") + if len(parts) < 2: + return {} + payload_b64 = parts[1] + payload_b64 += "=" * (-len(payload_b64) % 4) + payload = json.loads(base64.urlsafe_b64decode(payload_b64)) + return payload if isinstance(payload, dict) else {} + except Exception: + return {} + + @staticmethod + def _token_days_remaining(token: str) -> float | None: + # Дней до истечения токена. + exp = OpenLaneAuthenticator._decode_jwt_exp(token) + if exp is None: + return None + now = datetime.now(timezone.utc).timestamp() + return (exp - now) / 86400 + + @staticmethod + def _token_seconds_remaining(token: str) -> int | None: + # Секунд до истечения токена. + exp = OpenLaneAuthenticator._decode_jwt_exp(token) + if exp is None: + return None + return int(exp - time.time()) + + @staticmethod + def _extract_cookie_value(context: BrowserContext, name: str) -> str | None: + # Извлекаем значение cookie по имени. + cookies = context.cookies("https://app.openlane.com") + for cookie in cookies: + if cookie.get("name") == name: + value = str(cookie.get("value") or "").strip() + if value: + return value + return None + + def _is_access_token_fresh(self, token: str) -> bool: + # Проверяем что access_token ещё жив. + remain = self._token_seconds_remaining(token) + return remain is not None and remain > _ACCESS_TOKEN_MIN_TTL_SECONDS + + def _extract_refresh_token_from_cookies(self, context: BrowserContext) -> str | None: + # Извлечение refresh_token из cookies. + return self._extract_cookie_value(context, "refresh_token") + + def _extract_access_token_from_cookies(self, context: BrowserContext) -> str | None: + # Извлечение access_token из cookies. + return self._extract_cookie_value(context, "access_token") + + def _refresh_access_token_via_okta(self, refresh_token: str) -> tuple[str | None, str | None]: + # Прямой refresh через Okta token endpoint. + if not refresh_token: + return None, None + if not self.config.okta_client_id: + logger.warning("OPENLANE_OKTA_CLIENT_ID is empty, direct refresh disabled") + return None, None + + form_data = urlencode( + { + "grant_type": "refresh_token", + "client_id": self.config.okta_client_id, + "redirect_uri": self.config.okta_redirect_uri, + "refresh_token": refresh_token, + } + ).encode("utf-8") + req = Request( + self.config.okta_token_endpoint, + data=form_data, + method="POST", + headers={ + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json", + }, + ) + + try: + with urlopen(req, timeout=self.config.okta_timeout_seconds) as resp: + payload = json.loads(resp.read().decode("utf-8")) + access_token = str(payload.get("access_token") or "").strip() + new_refresh_token = str(payload.get("refresh_token") or "").strip() or refresh_token + if not access_token: + logger.warning("Okta refresh response has no access_token") + return None, None + logger.info("Okta direct refresh succeeded") + return access_token, new_refresh_token + except HTTPError as exc: + try: + body = exc.read().decode("utf-8", errors="ignore")[:300] + except Exception: + body = "" + logger.warning("Okta direct refresh failed: HTTP %s %s", exc.code, body) + return None, None + except URLError as exc: + logger.warning("Okta direct refresh failed: %s", exc) + return None, None + except Exception as exc: + logger.warning("Okta direct refresh failed: %s", exc) + return None, None + + def _persist_access_token_to_storage_state(self, context: BrowserContext, access_token: str) -> None: + # Сохраняем access_token cookie в контекст. + expires = self._decode_jwt_exp(access_token) + if expires is None: + expires = int(time.time()) + 7200 + context.add_cookies( + [ + { + "name": "access_token", + "value": access_token, + "domain": ".openlane.com", + "path": "/", + "httpOnly": True, + "secure": True, + "sameSite": "None", + "expires": int(expires), + } + ] + ) + + def _maybe_rotate_token(self, context: BrowserContext) -> None: + # Проверка и ротация refresh_token. + new_token = self._extract_refresh_token_from_cookies(context) + if not new_token: + logger.debug("No refresh_token cookie found after auth — nothing to rotate") + return + + old_token = self.config.refresh_token or "" + + # Проверяем изменился ли токен. + if new_token != old_token: + days = self._token_days_remaining(new_token) + logger.info( + "refresh_token CHANGED (new exp: %.1f days). Persisting to .env", + days if days is not None else -1, + ) + self.config.refresh_token = new_token + self._persist_token_to_dotenv(new_token) + return + + # Токен не изменился — проверяем срок. + days = self._token_days_remaining(new_token) + if days is not None: + logger.info("refresh_token unchanged, %.1f days remaining", days) + if days < _TOKEN_REFRESH_THRESHOLD_DAYS: + logger.warning( + "⚠ refresh_token expires in %.1f days! " + "Run `openlane-login` or set OPENLANE_USERNAME + OPENLANE_PASSWORD " + "to auto-renew on next sync.", + days, + ) + + @staticmethod + def _persist_token_to_dotenv(token: str) -> None: + # Запись refresh_token в .env. + env_path = Path(".env") + if not env_path.exists(): + # Создаём .env с токеном. + env_path.write_text( + f"OPENLANE_REFRESH_TOKEN={token}\n", + encoding="utf-8", + ) + logger.info("Created .env with rotated OPENLANE_REFRESH_TOKEN") + return + + content = env_path.read_text(encoding="utf-8") + pattern = re.compile(r"^OPENLANE_REFRESH_TOKEN=.*$", re.MULTILINE) + replacement = f"OPENLANE_REFRESH_TOKEN={token}" + + if pattern.search(content): + new_content = pattern.sub(replacement, content) + else: + new_content = content.rstrip("\n") + f"\n{replacement}\n" + + env_path.write_text(new_content, encoding="utf-8") + # Обновляем env процесса. + os.environ["OPENLANE_REFRESH_TOKEN"] = token + logger.info("Persisted rotated OPENLANE_REFRESH_TOKEN to .env") + + def bootstrap_authenticated_context(self, context: BrowserContext) -> Page: + # Восстановление сессии: storage_state → login → ошибка. + storage_state_path = Path(self.config.storage_state_file) + + if storage_state_path.exists(): + logger.info("OpenLane: restoring session from %s", storage_state_path) + page = context.new_page() + page.set_default_timeout(self.config.request_timeout_ms) + # Блокируем трекинг/картинки — нужен только fetch(). + BrowserFactory.enable_resource_blocking(page) + + # Проверка cookies без сетевых запросов. + cookies = context.cookies("https://app.openlane.com") + cookie_names = {c["name"] for c in cookies} + access_token = self._extract_access_token_from_cookies(context) + if access_token and self._is_access_token_fresh(access_token): + logger.info( + "OpenLane session restored: access_token cookie present (%d cookies total)", + len(cookies), + ) + # Лёгкий API endpoint для установки origin в браузере. + page.goto( + "https://app.openlane.com/api/_next/time", + wait_until="domcontentloaded", + ) + return page + + # Пробуем прямой refresh access_token без навигации. + refresh_token = self._extract_refresh_token_from_cookies(context) or self.config.refresh_token + if refresh_token: + new_access_token, new_refresh_token = self._refresh_access_token_via_okta(refresh_token) + if new_access_token: + self._persist_access_token_to_storage_state(context, new_access_token) + if new_refresh_token and new_refresh_token != (self.config.refresh_token or ""): + self.config.refresh_token = new_refresh_token + self._persist_token_to_dotenv(new_refresh_token) + self._persist_storage_state(context) + page.goto( + "https://app.openlane.com/api/_next/time", + wait_until="domcontentloaded", + ) + logger.info("OpenLane session restored via direct Okta refresh") + return page + logger.warning("Direct refresh failed, trying sign_in fallback") + + # Если refresh_token был только в env, добавляем cookie вручную. + if "refresh_token" not in cookie_names and self.config.refresh_token: + self._inject_refresh_token(context) + cookies = context.cookies("https://app.openlane.com") + cookie_names = {c["name"] for c in cookies} + + logger.warning("storage_state exists but no access_token cookie — trying navigation") + # Fallback: SPA обменяет refresh_token на access_token. + if "refresh_token" in cookie_names: + try: + page.goto(self.config.sign_in_url, wait_until="domcontentloaded") + self._wait_for_authenticated_session(page) + self._persist_storage_state(context) + self._maybe_rotate_token(context) + return page + except OpenLaneAuthError: + logger.warning("SPA refresh_token exchange failed") + page.close() + + # Нет storage_state — пробуем логин. + if self.config.username and self.config.password: + return self.login(context) + + raise OpenLaneAuthError( + "No valid session found. Either:\n" + " 1. Run `python scripts/explore_site.py` to login manually and save storage_state, or\n" + " 2. Set OPENLANE_USERNAME and OPENLANE_PASSWORD in .env" + ) + + def login(self, context: BrowserContext) -> Page: + self.validate_credentials() + page = context.new_page() + page.set_default_timeout(self.config.request_timeout_ms) + logger.info("OpenLane login: opening sign-in page %s", self.config.sign_in_url) + page.goto(self.config.sign_in_url, wait_until="domcontentloaded") + self._fill_login_form(page) + self._wait_for_authenticated_session(page) + self._persist_storage_state(context) + self._maybe_rotate_token(context) + logger.info("OpenLane login successful") + return page + + def _fill_login_form(self, page: Page) -> None: + email_selectors = [ + 'input[name="username"]', + 'input[name="user[login]"]', + 'input[name="email"]', + 'input[name="user[email]"]', + 'input[autocomplete="username"]', + 'input[type="email"]', + 'input[type="text"]', + '#email', + ] + password_selectors = [ + 'input[name="password"]', + 'input[name="user[password]"]', + 'input[type="password"]', + '#password', + ] + submit_selectors = [ + 'button[type="submit"]', + 'input[type="submit"]', + 'button:has-text("Sign in")', + 'button:has-text("Log in")', + ] + + email_filled = self._fill_first(page, email_selectors, self.config.username) + password_filled = self._fill_first(page, password_selectors, self.config.password) + if not email_filled or not password_filled: + raise OpenLaneAuthError("OpenLane login form fields were not found") + + if not self._click_first(page, submit_selectors): + raise OpenLaneAuthError("OpenLane login submit button was not found") + + def _wait_for_authenticated_session(self, page: Page) -> None: + # Ждём пока SPA обменяет refresh_token → access_token. + try: + page.wait_for_url( + lambda url: "/sign_in" not in url.lower(), + timeout=min(self.config.request_timeout_ms, 30000), + ) + except PlaywrightTimeoutError: + raise OpenLaneAuthError( + "OpenLane authentication failed: page stayed on sign_in " + "(refresh_token likely invalid or expired)" + ) + + # Проверяем что не на странице ошибки. + current_url = page.url.lower() + logger.debug("After auth redirect, URL: %s", page.url) + if "/sign_in" in current_url: + raise OpenLaneAuthError("OpenLane authentication failed: still on sign_in page") + + # Ждём появления access_token cookie. + for attempt in range(15): + cookies = page.context.cookies("https://app.openlane.com") + cookie_names = {c["name"] for c in cookies} + if "access_token" in cookie_names: + logger.info("OpenLane authenticated: access_token cookie present (attempt %d)", attempt) + return + page.wait_for_timeout(500) + + # Нет access_token cookie, но URL не sign_in — продолжаем. + logger.warning( + "access_token cookie not found after redirect, but page is at %s — continuing", + page.url, + ) + + def _persist_storage_state(self, context: BrowserContext) -> None: + if not self.config.persist_storage_state: + return + storage_state_path = Path(self.config.storage_state_file) + storage_state_path.parent.mkdir(parents=True, exist_ok=True) + context.storage_state(path=str(storage_state_path)) + logger.info("OpenLane storage state saved to %s", storage_state_path) + + @staticmethod + def _fill_first(page: Page, selectors: list[str], value: str) -> bool: + for selector in selectors: + locator = page.locator(selector) + if locator.count() == 0: + continue + try: + locator.first.fill(value) + return True + except PlaywrightTimeoutError: + continue + return False + + @staticmethod + def _click_first(page: Page, selectors: list[str]) -> bool: + for selector in selectors: + locator = page.locator(selector) + if locator.count() == 0: + continue + try: + locator.first.click() + return True + except PlaywrightTimeoutError: + continue + return False diff --git a/openlane_scraper/openlane/checkpoint.py b/openlane_scraper/openlane/checkpoint.py new file mode 100644 index 0000000..f9cb40c --- /dev/null +++ b/openlane_scraper/openlane/checkpoint.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field +from pathlib import Path + + +@dataclass(slots=True) +class OpenLaneCheckpoint: + max_pages: int + completed_pages: list[int] = field(default_factory=list) + failed_pages: list[int] = field(default_factory=list) + total_records: int = 0 + last_saved_at: str | None = None + + @property + def completed_set(self) -> set[int]: + return set(self.completed_pages) + + +class OpenLaneCheckpointStore: + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + + def load(self, max_pages: int) -> OpenLaneCheckpoint: + if not self.path.exists(): + return OpenLaneCheckpoint(max_pages=max_pages) + data = json.loads(self.path.read_text(encoding="utf-8")) + return OpenLaneCheckpoint( + max_pages=int(data.get("max_pages") or max_pages), + completed_pages=sorted({int(page) for page in data.get("completed_pages", [])}), + failed_pages=sorted({int(page) for page in data.get("failed_pages", [])}), + total_records=int(data.get("total_records") or 0), + last_saved_at=data.get("last_saved_at"), + ) + + def save(self, checkpoint: OpenLaneCheckpoint) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + payload = asdict(checkpoint) + self.path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True), + encoding="utf-8", + ) diff --git a/openlane_scraper/openlane/client.py b/openlane_scraper/openlane/client.py new file mode 100644 index 0000000..689624b --- /dev/null +++ b/openlane_scraper/openlane/client.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import base64 +import json +import logging +import random +import time +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlencode + +from playwright.sync_api import Page + +from ..core.config import OpenLaneConfig + +logger = logging.getLogger("openlane_scraper.openlane.client") + + +class OpenLaneRequestError(RuntimeError): + def __init__(self, message: str, *, status_code: int | None = None) -> None: + super().__init__(message) + self.status_code = status_code + + +@dataclass(slots=True) +class OpenLanePageResult: + page: int + records: list[dict[str, Any]] + raw_payload: dict[str, Any] + status_code: int + + +def _decode_access_token(token: str) -> dict[str, Any]: + # Декодирование JWT payload (без проверки подписи). + parts = token.split(".") + if len(parts) < 2: + return {} + payload_b64 = parts[1] + "=" * (-len(parts[1]) % 4) + try: + return json.loads(base64.urlsafe_b64decode(payload_b64)) + except Exception: + return {} + + +class OpenLaneClient: + # HTTP-клиент OpenLane search API через Playwright. + + def __init__(self, authenticated_page: Page, config: OpenLaneConfig) -> None: + self.page = authenticated_page + self.config = config + self._user_id: str | None = None + self._dealership_id: str | None = None + self._init_ids_from_cookies() + + def _init_ids_from_cookies(self) -> None: + # Извлекаем user_id и dealership_id из access_token. + cookies = self.page.context.cookies("https://app.openlane.com") + for cookie in cookies: + if cookie.get("name") == "access_token": + payload = _decode_access_token(cookie["value"]) + self._user_id = str(payload.get("user_id", "")) + on_behalf = payload.get("on_behalf_of", {}) + self._dealership_id = str(on_behalf.get("dealership_id", "")) + if self._user_id and self._dealership_id: + logger.info( + "OpenLane client: user_id=%s dealership_id=%s", + self._user_id, self._dealership_id, + ) + return + logger.warning("OpenLane client: access_token cookie not found, API requests may fail") + + def _build_headers(self) -> dict[str, str]: + # Заголовки как у фронтенда OpenLane. + headers: dict[str, str] = { + "Accept": "application/json, application/vnd.backlotcars.v3", + "application": "webapp", + } + if self._user_id: + headers["x-rbz-user-id"] = self._user_id + if self._dealership_id: + headers["x-rbz-dealership-id"] = self._dealership_id + return headers + + def fetch_page(self, page: int) -> OpenLanePageResult: + params = { + "page": page, + "source_tab": self.config.source_tab, + "sale_types": self.config.sale_types, + } + if self.config.page_size > 0: + params["per_page"] = self.config.page_size + + self._sleep_with_jitter() + url = f"{self.config.api_search_url}?{urlencode(params)}" + logger.debug("OpenLane fetch page=%s url=%s", page, url) + + # fetch() с авторизацией в браузере. + headers_js = json.dumps(self._build_headers()) + fetch_result = self.page.evaluate( + """async ([url, headersJson]) => { + const headers = JSON.parse(headersJson); + const resp = await fetch(url, { + method: 'GET', + credentials: 'include', + headers: headers + }); + const text = await resp.text(); + return { status: resp.status, body: text }; + }""", + [url, headers_js], + ) + return self._parse_response(page, fetch_result) + + def fetch_vehicle_images(self, vehicle_id: str | int) -> list[dict[str, Any]]: + """Загружает изображения автомобиля через `/api/vehicles/{id}/images`.""" + if vehicle_id is None: + return [] + + vehicle_id_str = str(vehicle_id).strip() + if not vehicle_id_str: + return [] + + self._sleep_with_jitter() + url = f"https://app.openlane.com/api/vehicles/{vehicle_id_str}/images" + logger.debug("OpenLane fetch images vehicle_id=%s", vehicle_id_str) + + headers_js = json.dumps(self._build_headers()) + fetch_result = self.page.evaluate( + """async ([url, headersJson]) => { + const headers = JSON.parse(headersJson); + const resp = await fetch(url, { + method: 'GET', + credentials: 'include', + headers: headers + }); + const text = await resp.text(); + return { status: resp.status, body: text }; + }""", + [url, headers_js], + ) + + status_code = int(fetch_result.get("status", 0)) + text = str(fetch_result.get("body", "")) + if status_code == 404: + return [] + if status_code == 403: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden", + status_code=status_code, + ) + if status_code == 429: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests", + status_code=status_code, + ) + if status_code >= 500: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}", + status_code=status_code, + ) + if status_code == 401: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated", + status_code=status_code, + ) + if status_code >= 400: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}", + status_code=status_code, + ) + + if not text.strip(): + return [] + + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON" + ) from exc + + return self._extract_vehicle_images(payload) + + def fetch_vehicle_images_batch(self, vehicle_ids: list[str | int]) -> dict[str, list[dict[str, Any]]]: + """Пакетно загружает изображения автомобилей через `/api/vehicles/{id}/images`. + + Возвращает словарь `vehicle_id -> список image-объектов`. + """ + normalized_ids: list[str] = [] + seen: set[str] = set() + for vehicle_id in vehicle_ids: + vehicle_id_str = str(vehicle_id).strip() + if not vehicle_id_str or vehicle_id_str in seen: + continue + seen.add(vehicle_id_str) + normalized_ids.append(vehicle_id_str) + + if not normalized_ids: + return {} + + self._sleep_with_jitter() + headers_js = json.dumps(self._build_headers()) + fetch_results = self.page.evaluate( + """async ([vehicleIds, headersJson]) => { + const headers = JSON.parse(headersJson); + const tasks = vehicleIds.map(async (vehicleId) => { + try { + const url = `https://app.openlane.com/api/vehicles/${vehicleId}/images`; + const resp = await fetch(url, { + method: 'GET', + credentials: 'include', + headers: headers, + }); + const text = await resp.text(); + return { vehicleId, status: resp.status, body: text }; + } catch (error) { + return { vehicleId, status: 0, body: '', error: String(error) }; + } + }); + return await Promise.all(tasks); + }""", + [normalized_ids, headers_js], + ) + + result_map: dict[str, list[dict[str, Any]]] = {} + for item in fetch_results: + vehicle_id_str = str(item.get("vehicleId", "")).strip() + status_code = int(item.get("status", 0)) + text = str(item.get("body", "")) + + if not vehicle_id_str: + continue + + if status_code == 404: + result_map[vehicle_id_str] = [] + continue + if status_code == 403: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 403 Forbidden", + status_code=status_code, + ) + if status_code == 429: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 429 Too Many Requests", + status_code=status_code, + ) + if status_code >= 500 or status_code == 0: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned server error {status_code}", + status_code=status_code if status_code else 500, + ) + if status_code == 401: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned 401; session is not authenticated", + status_code=status_code, + ) + if status_code >= 400: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned unexpected status {status_code}", + status_code=status_code, + ) + + if not text.strip(): + result_map[vehicle_id_str] = [] + continue + + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + raise OpenLaneRequestError( + f"OpenLane images vehicle_id={vehicle_id_str} returned invalid JSON" + ) from exc + + result_map[vehicle_id_str] = self._extract_vehicle_images(payload) + + return result_map + + def _parse_response(self, page: int, fetch_result: dict[str, Any]) -> OpenLanePageResult: + # Парсинг ответа fetch(): {status, body}. + status_code = int(fetch_result.get("status", 0)) + text = str(fetch_result.get("body", "")) + logger.debug( + "OpenLane response: page=%s status=%s body_len=%d body_preview=%.500s", + page, status_code, len(text), text[:500], + ) + if status_code == 403: + raise OpenLaneRequestError( + f"OpenLane page={page} returned 403 Forbidden — possible block, stopping", + status_code=status_code, + ) + if status_code == 429: + raise OpenLaneRequestError( + f"OpenLane page={page} returned 429 Too Many Requests — rate limited", + status_code=status_code, + ) + if status_code >= 500: + raise OpenLaneRequestError( + f"OpenLane page={page} returned server error {status_code}", + status_code=status_code, + ) + if status_code == 401: + raise OpenLaneRequestError( + f"OpenLane page={page} returned 401; session is not authenticated", + status_code=status_code, + ) + if status_code >= 400: + raise OpenLaneRequestError( + f"OpenLane page={page} returned unexpected status {status_code}", + status_code=status_code, + ) + + try: + payload = json.loads(text) + except json.JSONDecodeError as exc: + raise OpenLaneRequestError(f"OpenLane page={page} returned invalid JSON") from exc + + records = self._extract_records(payload) + logger.debug( + "OpenLane extracted: page=%s records=%d top_keys=%s", + page, len(records), list(payload.keys())[:10], + ) + return OpenLanePageResult( + page=page, + records=records, + raw_payload=payload, + status_code=status_code, + ) + + @staticmethod + def _extract_records(payload: dict[str, Any]) -> list[dict[str, Any]]: + candidates = [ + payload.get("results"), + payload.get("items"), + payload.get("data"), + payload.get("vehicles"), + ] + for candidate in candidates: + if isinstance(candidate, list): + return [item for item in candidate if isinstance(item, dict)] + if isinstance(candidate, dict): + nested_candidates = [ + candidate.get("results"), + candidate.get("items"), + candidate.get("vehicles"), + ] + for nested in nested_candidates: + if isinstance(nested, list): + return [item for item in nested if isinstance(item, dict)] + return [] + + @staticmethod + def _extract_vehicle_images(payload: Any) -> list[dict[str, Any]]: + candidates: list[Any] = [] + if isinstance(payload, dict): + candidates.extend( + [ + payload.get("vehicle_images"), + payload.get("images"), + payload.get("data"), + ] + ) + data = payload.get("data") + if isinstance(data, dict): + candidates.extend([data.get("vehicle_images"), data.get("images")]) + elif isinstance(payload, list): + candidates.append(payload) + + for candidate in candidates: + if isinstance(candidate, list): + return [item for item in candidate if isinstance(item, dict)] + return [] + + def _sleep_with_jitter(self) -> None: + low = min(self.config.throttle_min_seconds, self.config.throttle_max_seconds) + high = max(self.config.throttle_min_seconds, self.config.throttle_max_seconds) + time.sleep(random.uniform(low, high)) diff --git a/openlane_scraper/openlane/mapper.py b/openlane_scraper/openlane/mapper.py new file mode 100644 index 0000000..2c250ae --- /dev/null +++ b/openlane_scraper/openlane/mapper.py @@ -0,0 +1,424 @@ +"""Маппер: JSON-запись OpenLane API → CarRecord для сохранения в БД.""" + +from __future__ import annotations + +import hashlib +import logging +import re +from datetime import datetime, timezone +from typing import Any + +from ..storage.schemas import CarRecord, ImageRecord + +logger = logging.getLogger("openlane_scraper.openlane.mapper") + +# Маппинг body_type из OpenLane в наш enum. +BODY_MAP: dict[str, str] = { + "sedan": "SEDAN", + "coupe": "COUPE", + "suv": "SUV", + "sport utility": "SUV", + "crossover": "SUV", + "hatchback": "HATCHBACK", + "minivan": "MINIVAN", + "van": "MINIVAN", + "wagon": "STATION_WAGON", + "station wagon": "STATION_WAGON", + "pickup": "PICKUP", + "truck": "TRUCK", + "convertible": "OPEN", + "cabriolet": "OPEN", + "roadster": "OPEN", + "rv": "RV", + "motorhome": "RV", +} + +DRIVE_MAP: dict[str, str] = { + "fwd": "FWD", + "front wheel drive": "FWD", + "front-wheel drive": "FWD", + "rwd": "RWD", + "rear wheel drive": "RWD", + "rear-wheel drive": "RWD", + "awd": "4WD", + "4wd": "4WD", + "all wheel drive": "4WD", + "all-wheel drive": "4WD", + "4x4": "4WD", + "2wd": "2WD", + "two wheel drive": "2WD", +} + +GEARBOX_MAP: dict[str, str] = { + "automatic": "AT", + "auto": "AT", + "at": "AT", + "manual": "MT", + "mt": "MT", + "cvt": "CVT", + "continuously variable": "CVT", + "electric": "EV", + "ev": "EV", +} + +_MILEAGE_RE = re.compile(r"[\d,]+") +_ENGINE_RE = re.compile(r"(\d+\.?\d*)\s*[lL]") + + +def _safe_str(value: Any, default: str = "") -> str: + if value is None: + return default + return str(value).strip() or default + + +def _safe_int(value: Any) -> int | None: + if value is None: + return None + try: + cleaned = str(value).replace(",", "").strip() + if not cleaned: + return None + return int(float(cleaned)) + except (ValueError, TypeError): + return None + + +def _normalize_enum(raw: Any, mapping: dict[str, str], default: str = "NA") -> str: + if not raw: + return default + key = str(raw).strip().lower() + return mapping.get(key, default) + + +def _generate_parser_id(origin_id: str) -> str: + return hashlib.sha256(origin_id.encode()).hexdigest()[:40] + + +def _generate_slug(year: int | None, brand: str, model: str, origin_id: str) -> str: + parts = [] + if year: + parts.append(str(year)) + parts.append(brand.lower()) + parts.append(model.lower()) + parts.append(origin_id.replace(":", "-")) + slug = "-".join(parts) + slug = re.sub(r"[^a-z0-9\-]", "-", slug) + slug = re.sub(r"-+", "-", slug).strip("-") + return slug[:200] + + +def _build_openlane_origin_id(raw_id: Any) -> str | None: + """Строит origin_id в формате openlane:id.""" + normalized = _safe_str(raw_id) + if not normalized: + return None + if normalized.lower().startswith("openlane:"): + normalized = _safe_str(normalized.split(":", 1)[1]) + if not normalized: + return None + return f"openlane:{normalized}" + + +def _extract_nested(record: dict[str, Any], *keys: str) -> Any: + """Извлекает значение из вложенного словаря по цепочке ключей.""" + obj: Any = record + for key in keys: + if isinstance(obj, dict): + obj = obj.get(key) + else: + return None + return obj + + +def _extract_images(record: dict[str, Any]) -> list[ImageRecord]: + """Извлекает изображения из записи OpenLane.""" + images: list[ImageRecord] = [] + seen_urls: set[str] = set() + + raw_images = ( + record.get("images") + or record.get("photos") + or record.get("media", {}).get("images") + or record.get("image_urls") + or [] + ) + + if isinstance(raw_images, list): + for idx, img in enumerate(raw_images): + if isinstance(img, str): + url = img.strip() + if url and url not in seen_urls: + seen_urls.add(url) + images.append(ImageRecord( + fullres_image=url, + preview_image=url, + order_index=idx, + )) + elif isinstance(img, dict): + fullres = _safe_str( + img.get("full") or img.get("fullres") or img.get("url") + or img.get("original") or img.get("large") or img.get("href") + or img.get("large_resolution_url") + ) + preview = _safe_str( + img.get("thumbnail") or img.get("thumb") or img.get("preview") + or img.get("small") or img.get("low_resolution_url") + or fullres + ) + if fullres and fullres not in seen_urls: + seen_urls.add(fullres) + images.append(ImageRecord( + fullres_image=fullres, + preview_image=preview, + order_index=idx, + )) + + # Fallback: одиночное изображение. + if not images: + single = ( + record.get("image_url") + or record.get("image") + or record.get("large_resolution_url") + or record.get("low_resolution_url") + or record.get("thumbnail") + or record.get("photo_url") + or _extract_nested(record, "media", "primary") + ) + if single and isinstance(single, str) and single.strip(): + images.append(ImageRecord( + fullres_image=single.strip(), + preview_image=single.strip(), + order_index=0, + )) + + return images + + +def _parse_mileage(raw: Any) -> int: + if raw is None: + return 0 + if isinstance(raw, (int, float)): + return max(0, int(raw)) + text = str(raw) + match = _MILEAGE_RE.search(text) + if match: + try: + return max(0, int(match.group().replace(",", ""))) + except ValueError: + pass + return 0 + + +def _parse_engine_volume_cc(raw: Any) -> int | None: + """Парсит объём двигателя и возвращает значение в кубических сантиметрах.""" + if raw is None: + return None + if isinstance(raw, (int, float)): + value = float(raw) + # Если значение < 20 — скорее всего это литры, конвертируем в cc. + if 0 < value < 20: + return int(value * 1000) + if value >= 100: + return int(value) + return None + text = str(raw) + match = _ENGINE_RE.search(text) + if match: + liters = float(match.group(1)) + return int(liters * 1000) + return None + + +def map_openlane_record(record: dict[str, Any]) -> CarRecord | None: + """Маппит одну запись из OpenLane API в CarRecord. + + Возвращает None, если запись не содержит минимально необходимых данных. + """ + # Извлекаем идентификатор. + raw_id = ( + record.get("id") + or record.get("vehicle_id") + or record.get("listing_id") + or record.get("vin") + ) + if not raw_id: + logger.debug("Skipping record without id: %s", record.get("vin", "unknown")) + return None + + origin_id = _build_openlane_origin_id(raw_id) + if not origin_id: + logger.debug("Skipping record with malformed id: %s", raw_id) + return None + + # Бренд и модель — обязательные поля. + brand = _safe_str( + record.get("make") + or record.get("brand") + or record.get("manufacturer") + or _extract_nested(record, "vehicle", "make") + ).upper() + + model = _safe_str( + record.get("model") + or record.get("model_name") + or _extract_nested(record, "vehicle", "model") + ).upper() + + if not brand or not model: + logger.debug("Skipping record without brand/model: %s", origin_id) + return None + + year = _safe_int( + record.get("year") + or record.get("model_year") + or _extract_nested(record, "vehicle", "year") + ) + + price = _safe_int( + record.get("price") + or record.get("current_bid") + or record.get("buy_now_price") + or record.get("asking_price") + or record.get("sale_price") + or _extract_nested(record, "pricing", "current") + or _extract_nested(record, "pricing", "buy_now") + ) + + currency = _safe_str( + record.get("currency") + or record.get("currency_code") + or _extract_nested(record, "pricing", "currency"), + "USD", + ).upper() + if currency not in {"JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD"}: + currency = "USD" + + mileage = _parse_mileage( + record.get("mileage") + or record.get("odometer") + or record.get("odometer_reading") + or _extract_nested(record, "vehicle", "mileage") + ) + + color = _safe_str( + record.get("color") + or record.get("exterior_color") + or _extract_nested(record, "vehicle", "color"), + "other", + ).lower() + + body_type = _normalize_enum( + record.get("body_type") + or record.get("body_style") + or record.get("vehicle_type") + or _extract_nested(record, "vehicle", "body_type"), + BODY_MAP, + "OTHER", + ) + + drive = _normalize_enum( + record.get("drive_type") + or record.get("drivetrain") + or record.get("drive") + or _extract_nested(record, "vehicle", "drivetrain"), + DRIVE_MAP, + ) + + gearbox = _normalize_enum( + record.get("transmission") + or record.get("gearbox") + or _extract_nested(record, "vehicle", "transmission"), + GEARBOX_MAP, + ) + + engine_volume = _parse_engine_volume_cc( + record.get("engine") + or record.get("engine_size") + or record.get("displacement") + or _extract_nested(record, "vehicle", "engine") + ) + + # URL записи на OpenLane. + origin_url = _safe_str( + record.get("url") + or record.get("listing_url") + or record.get("detail_url") + or record.get("permalink") + ) + if not origin_url: + origin_url = f"https://app.openlane.com/vehicles/{raw_id}" + + country = _safe_str( + record.get("country") + or record.get("location_country") + or _extract_nested(record, "location", "country"), + "US", + ).upper() + if country not in {"JP", "KR", "US", "CA", "NA"}: + country = "US" + + is_damaged = bool( + record.get("is_damaged") + or record.get("has_damage") + or record.get("damage_type") + ) + + vin = _safe_str(record.get("vin") or _extract_nested(record, "vehicle", "vin")) + evaluation = vin if vin else None + + selling_type = "AUCTION" + sale_type = _safe_str(record.get("sale_type") or record.get("listing_type")).lower() + if sale_type in {"buy_now", "fixed_price", "stock"}: + selling_type = "STOCK" + elif sale_type in {"tender"}: + selling_type = "TENDER" + + images = _extract_images(record) + + parser_id = _generate_parser_id(origin_id) + slug = _generate_slug(year, brand, model, origin_id) + + return CarRecord( + parser_id=parser_id, + brand=brand, + model=model, + year=year, + price=price, + currency=currency, + mileage=mileage, + country=country, + is_sold=False, + color=color, + drive=drive if drive != "NA" else None, + gearbox=gearbox if gearbox != "NA" else None, + body_type=body_type, + engine_volume=engine_volume, + selling_type=selling_type, + origin="OPENLANE", + origin_url=origin_url, + origin_id=origin_id, + is_damaged=is_damaged, + evaluation=evaluation, + slug=slug, + images=images, + ) + + +def map_openlane_records(records: list[dict[str, Any]]) -> list[CarRecord]: + """Маппит список записей OpenLane API в список CarRecord. + + Пропускает записи без минимально необходимых данных. + """ + result: list[CarRecord] = [] + for record in records: + try: + car = map_openlane_record(record) + if car is not None: + result.append(car) + except Exception: + logger.warning( + "Failed to map OpenLane record id=%s", + record.get("id", "unknown"), + exc_info=True, + ) + return result diff --git a/openlane_scraper/openlane/runner.py b/openlane_scraper/openlane/runner.py new file mode 100644 index 0000000..8bbc021 --- /dev/null +++ b/openlane_scraper/openlane/runner.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +import logging +import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from playwright.sync_api import sync_playwright + +from ..browser.factory import BrowserFactory +from ..core.config import Settings +from ..core.logs import set_trace_id, setup_logging +from .auth import OpenLaneAuthenticator +from .checkpoint import OpenLaneCheckpoint, OpenLaneCheckpointStore +from .client import OpenLaneClient, OpenLanePageResult, OpenLaneRequestError +from .writer import OpenLaneResultWriter + +logger = logging.getLogger("openlane_scraper.openlane.runner") + + +@dataclass(slots=True) +class OpenLaneScrapeResult: + jsonl_path: str + aggregated_path: str + checkpoint_path: str + storage_state_path: str + completed_pages: list[int] + failed_pages: list[int] + total_records: int + elapsed_seconds: float + + +class OpenLaneScrapeRunner: + def __init__(self, runtime_settings: Settings | None = None) -> None: + self.settings = runtime_settings or Settings() + setup_logging(self.settings.log_level, self.settings.log_file) + self.trace_id = f"openlane-{uuid.uuid4().hex[:8]}" + set_trace_id(self.trace_id) + self.openlane = self.settings.openlane + self.browser_factory = BrowserFactory(self.settings) + self.authenticator = OpenLaneAuthenticator(self.openlane) + self.writer = OpenLaneResultWriter( + self.openlane.jsonl_output, + self.openlane.aggregated_output, + ) + self.checkpoint_store = OpenLaneCheckpointStore(self.openlane.checkpoint_file) + + def run( + self, + *, + max_pages: int | None = None, + concurrency: int | None = None, + resume: bool = False, + checkpoint_every_pages: int | None = None, + ) -> OpenLaneScrapeResult: + started_at = time.perf_counter() + max_pages = max_pages or self.openlane.max_pages + concurrency = max(1, min(concurrency or self.openlane.concurrency, 5)) + checkpoint_every_pages = checkpoint_every_pages or self.openlane.checkpoint_every_pages + + checkpoint = self.checkpoint_store.load(max_pages=max_pages) if resume else OpenLaneCheckpoint(max_pages=max_pages) + checkpoint.max_pages = max_pages + + if not resume: + self._reset_outputs() + + logger.info( + "Starting OpenLane scrape: max_pages=%s concurrency=%s resume=%s checkpoint_every_pages=%s", + max_pages, + concurrency, + resume, + checkpoint_every_pages, + ) + + with sync_playwright() as playwright: + browser = self.browser_factory.create_browser(playwright) + try: + auth_pages = [] + for worker_index in range(concurrency): + storage_state_path = self.openlane.storage_state_file if Path(self.openlane.storage_state_file).exists() else None + context = self.browser_factory.create_context(browser, storage_state_path=storage_state_path) + auth_page = self.authenticator.bootstrap_authenticated_context(context) + auth_pages.append(auth_page) + logger.info("OpenLane worker session initialized: worker=%s", worker_index + 1) + + pending_pages = [ + page for page in range(1, max_pages + 1) + if page not in checkpoint.completed_set + ] + self._run_workers(auth_pages, pending_pages, checkpoint, checkpoint_every_pages) + finally: + browser.close() + + self.checkpoint_store.save(checkpoint) + aggregated = self.writer.finalize( + max_pages=max_pages, + completed_pages=checkpoint.completed_pages, + failed_pages=checkpoint.failed_pages, + ) + elapsed_seconds = time.perf_counter() - started_at + logger.info( + "OpenLane scrape finished: completed_pages=%s failed_pages=%s total_records=%s elapsed=%.2fs", + len(set(checkpoint.completed_pages)), + len(set(checkpoint.failed_pages)), + aggregated["total_records"], + elapsed_seconds, + ) + return OpenLaneScrapeResult( + jsonl_path=str(Path(self.openlane.jsonl_output)), + aggregated_path=str(Path(self.openlane.aggregated_output)), + checkpoint_path=str(Path(self.openlane.checkpoint_file)), + storage_state_path=str(Path(self.openlane.storage_state_file)), + completed_pages=sorted(set(checkpoint.completed_pages)), + failed_pages=sorted(set(checkpoint.failed_pages)), + total_records=int(aggregated["total_records"]), + elapsed_seconds=elapsed_seconds, + ) + + def _run_workers( + self, + auth_pages: list, + pending_pages: list[int], + checkpoint: OpenLaneCheckpoint, + checkpoint_every_pages: int, + ) -> None: + # Последовательная обработка страниц (sync API — однопоточный). + started_at = time.perf_counter() + + for idx, page_num in enumerate(pending_pages): + worker_index = (idx % len(auth_pages)) + 1 + auth_page = auth_pages[idx % len(auth_pages)] + + try: + result = self._fetch_page_with_retry(auth_page, page_num, worker_index) + self._handle_success(result, checkpoint, started_at) + except Exception as exc: + self._handle_failure(page_num, exc, checkpoint, started_at) + + processed_count = len(set(checkpoint.completed_pages)) + len(set(checkpoint.failed_pages)) + if processed_count and processed_count % checkpoint_every_pages == 0: + checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat() + self.checkpoint_store.save(checkpoint) + logger.info( + "OpenLane checkpoint saved: processed=%s completed=%s failed=%s", + processed_count, + len(set(checkpoint.completed_pages)), + len(set(checkpoint.failed_pages)), + ) + + def _fetch_page_with_retry(self, authenticated_page, page: int, worker_index: int) -> OpenLanePageResult: + set_trace_id(f"{self.trace_id}-w{worker_index}-p{page}") + client = OpenLaneClient(authenticated_page, self.openlane) + retry_schedule = self.openlane.retry_schedule_seconds + + for attempt in range(len(retry_schedule) + 1): + try: + logger.debug("OpenLane fetch attempt: page=%s worker=%s attempt=%s", page, worker_index, attempt + 1) + return client.fetch_page(page) + except OpenLaneRequestError as exc: + is_retryable = exc.status_code in {403, 429} or (exc.status_code is not None and exc.status_code >= 500) + if not is_retryable or attempt >= len(retry_schedule): + logger.error( + "OpenLane fetch failed permanently: page=%s worker=%s status=%s error=%s", + page, + worker_index, + exc.status_code, + exc, + ) + raise + delay = retry_schedule[attempt] + logger.warning( + "OpenLane retry scheduled: page=%s worker=%s status=%s delay=%.1fs attempt=%s", + page, + worker_index, + exc.status_code, + delay, + attempt + 1, + ) + time.sleep(delay) + except Exception: + if attempt >= len(retry_schedule): + raise + delay = retry_schedule[attempt] + logger.warning( + "OpenLane transient error retry: page=%s worker=%s delay=%.1fs attempt=%s", + page, + worker_index, + delay, + attempt + 1, + exc_info=True, + ) + time.sleep(delay) + + raise RuntimeError(f"OpenLane page {page} exhausted retries") + + def _handle_success( + self, + result: OpenLanePageResult, + checkpoint: OpenLaneCheckpoint, + started_at: float, + ) -> None: + written = self.writer.append_page(result.page, result.records) + checkpoint.completed_pages = sorted(set(checkpoint.completed_pages) | {result.page}) + checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) - {result.page}) + checkpoint.total_records += written + checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat() + self._log_progress(result.page, checkpoint, started_at, written, None) + + def _handle_failure( + self, + page: int, + exc: Exception, + checkpoint: OpenLaneCheckpoint, + started_at: float, + ) -> None: + checkpoint.failed_pages = sorted(set(checkpoint.failed_pages) | {page}) + checkpoint.last_saved_at = datetime.now(timezone.utc).isoformat() + self._log_progress(page, checkpoint, started_at, 0, exc) + + def _log_progress( + self, + page: int, + checkpoint: OpenLaneCheckpoint, + started_at: float, + records_written: int, + exc: Exception | None, + ) -> None: + completed = len(set(checkpoint.completed_pages)) + failed = len(set(checkpoint.failed_pages)) + elapsed_seconds = max(time.perf_counter() - started_at, 0.001) + pages_per_minute = (completed + failed) / elapsed_seconds * 60.0 + if exc is None: + logger.info( + "OpenLane progress: page=%s completed=%s failed=%s records=%s total_records=%s speed=%.2f pages/min", + page, + completed, + failed, + records_written, + checkpoint.total_records, + pages_per_minute, + ) + else: + logger.error( + "OpenLane page error: page=%s completed=%s failed=%s total_records=%s speed=%.2f pages/min error=%s", + page, + completed, + failed, + checkpoint.total_records, + pages_per_minute, + exc, + ) + + def interactive_login(self) -> str: + setup_logging(self.settings.log_level, self.settings.log_file) + with sync_playwright() as playwright: + browser = self.browser_factory.create_browser(playwright) + try: + context = self.browser_factory.create_context(browser) + return self.authenticator.interactive_login_and_persist(context) + finally: + browser.close() + + def _reset_outputs(self) -> None: + for raw_path in ( + self.openlane.jsonl_output, + self.openlane.aggregated_output, + self.openlane.checkpoint_file, + ): + path = Path(raw_path) + if path.exists(): + path.unlink() diff --git a/openlane_scraper/openlane/writer.py b/openlane_scraper/openlane/writer.py new file mode 100644 index 0000000..d6d79f8 --- /dev/null +++ b/openlane_scraper/openlane/writer.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +class OpenLaneResultWriter: + def __init__(self, jsonl_path: str | Path, aggregated_path: str | Path) -> None: + self.jsonl_path = Path(jsonl_path) + self.aggregated_path = Path(aggregated_path) + + def ensure_parent_dirs(self) -> None: + self.jsonl_path.parent.mkdir(parents=True, exist_ok=True) + self.aggregated_path.parent.mkdir(parents=True, exist_ok=True) + + def append_page(self, page: int, records: list[dict[str, Any]]) -> int: + self.ensure_parent_dirs() + written = 0 + with self.jsonl_path.open("a", encoding="utf-8") as fh: + for record in records: + payload = { + "page": page, + "record": record, + } + fh.write(json.dumps(payload, ensure_ascii=False) + "\n") + written += 1 + return written + + def finalize(self, *, max_pages: int, completed_pages: list[int], failed_pages: list[int]) -> dict[str, Any]: + self.ensure_parent_dirs() + records: list[dict[str, Any]] = [] + if self.jsonl_path.exists(): + with self.jsonl_path.open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + item = json.loads(line) + records.append(item) + + summary = { + "max_pages": max_pages, + "completed_pages": sorted(completed_pages), + "failed_pages": sorted(failed_pages), + "total_pages_completed": len(set(completed_pages)), + "total_pages_failed": len(set(failed_pages)), + "total_records": len(records), + "items": records, + } + self.aggregated_path.write_text( + json.dumps(summary, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + return summary diff --git a/openlane_scraper/scraper.py b/openlane_scraper/scraper.py new file mode 100644 index 0000000..e14d3e3 --- /dev/null +++ b/openlane_scraper/scraper.py @@ -0,0 +1,514 @@ +"""Главный оркестратор скрапинга OpenLane с сохранением в БД. + +Использует OpenLane API для получения данных и PersistenceService для записи в PostgreSQL. +Batched upsert, SyncRun-аудит, early-stop при отсутствии новых записей. +""" + +from __future__ import annotations + +import logging +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from playwright.sync_api import sync_playwright + +from .browser.factory import BrowserFactory +from .core.config import Settings +from .core.logs import set_trace_id, setup_logging +from .core.runtime_config import RuntimeConfig, apply_filters, load_runtime_config +from .openlane.auth import OpenLaneAuthenticator +from .openlane.client import OpenLaneClient, OpenLanePageResult, OpenLaneRequestError +from .openlane.mapper import map_openlane_records +from .storage.db import PersistenceService +from .storage.schemas import CarRecord, ImageRecord + +logger = logging.getLogger("openlane_scraper.scraper") + + +class OpenLaneScraper: + """Оркестратор: OpenLane API → маппинг → DB upsert.""" + + def __init__(self, runtime_settings: Settings | None = None) -> None: + self.settings = runtime_settings or Settings() + setup_logging(self.settings.log_level, self.settings.log_file) + self.trace_id = f"openlane-sync-{uuid.uuid4().hex[:8]}" + set_trace_id(self.trace_id) + self.openlane_config = self.settings.openlane + self.browser_factory = BrowserFactory(self.settings) + self.authenticator = OpenLaneAuthenticator(self.openlane_config) + self.persistence = PersistenceService(self.settings) + self.runtime_config = load_runtime_config(self.settings.runtime_config_file) + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + @staticmethod + def _normalize_origin_id(raw_id: Any) -> str | None: + if raw_id is None: + return None + normalized = str(raw_id).strip() + if not normalized: + return None + if normalized.lower().startswith("openlane:"): + normalized = normalized.split(":", 1)[1].strip() + if not normalized: + return None + return f"openlane:{normalized}" + + @staticmethod + def _build_image_records(images_payload: list[dict[str, Any]]) -> list[ImageRecord]: + images: list[ImageRecord] = [] + seen: set[str] = set() + for idx, img in enumerate(images_payload): + fullres = str( + img.get("url") + or img.get("large_resolution_url") + or img.get("image_large") + or img.get("image") + or img.get("full") + or img.get("fullres") + or "" + ).strip() + if not fullres or fullres in seen: + continue + preview = str( + img.get("low_resolution_url") + or img.get("medium_resolution_url") + or img.get("thumbnail_url") + or img.get("thumbnail") + or img.get("thumb") + or img.get("image") + or fullres + ).strip() + seen.add(fullres) + images.append( + ImageRecord( + fullres_image=fullres, + preview_image=preview or fullres, + order_index=idx, + ) + ) + return images + + @staticmethod + def _merge_image_records(existing: list[ImageRecord], fetched: list[ImageRecord]) -> list[ImageRecord]: + merged: list[ImageRecord] = [] + seen: set[str] = set() + + for source in (fetched, existing): + for image in source: + key = image.fullres_image.strip() + if not key or key in seen: + continue + seen.add(key) + merged.append(image) + + for idx, image in enumerate(merged): + image.order_index = idx + + return merged + + def _fetch_vehicle_images_with_retry( + self, + client: OpenLaneClient, + vehicle_id: str, + ) -> list[dict[str, Any]]: + retry_schedule = self.openlane_config.retry_schedule_seconds + for attempt in range(len(retry_schedule) + 1): + try: + return client.fetch_vehicle_images(vehicle_id) + except OpenLaneRequestError as exc: + if exc.status_code in {401, 403}: + raise + is_retryable = exc.status_code == 429 or ( + exc.status_code is not None and exc.status_code >= 500 + ) + if not is_retryable or attempt >= len(retry_schedule): + raise + delay = retry_schedule[attempt] + if exc.status_code == 429: + delay = max(delay * 2, 8.0) + logger.warning( + "Retry vehicle images vehicle_id=%s status=%s delay=%.1fs attempt=%d/%d", + vehicle_id, + exc.status_code, + delay, + attempt + 1, + len(retry_schedule), + ) + time.sleep(delay) + + return [] + + def _enrich_car_records_with_images( + self, + authenticated_page, + raw_records: list[dict[str, Any]], + car_records: list[CarRecord], + ) -> int: + if not car_records: + return 0 + + raw_by_origin: dict[str, dict[str, Any]] = {} + for raw in raw_records: + raw_id = ( + raw.get("id") + or raw.get("vehicle_id") + or raw.get("listing_id") + or raw.get("vin") + ) + origin_id = self._normalize_origin_id(raw_id) + if origin_id: + raw_by_origin[origin_id] = raw + + client = OpenLaneClient(authenticated_page, self.openlane_config) + images_added = 0 + + cars_by_vehicle_id: dict[str, CarRecord] = {} + for car in car_records: + raw = raw_by_origin.get(car.origin_id, {}) + vehicle_id = raw.get("id") or raw.get("vehicle_id") or raw.get("listing_id") + if not vehicle_id: + # fallback: пробуем id из origin_id. + vehicle_id = car.origin_id.split(":", 1)[1] if ":" in car.origin_id else None + if not vehicle_id: + continue + cars_by_vehicle_id[str(vehicle_id)] = car + + if not cars_by_vehicle_id: + return 0 + + vehicle_ids = list(cars_by_vehicle_id.keys()) + try: + images_map = self._fetch_vehicle_images_batch_with_retry(client, vehicle_ids) + except OpenLaneRequestError as exc: + if exc.status_code in {401, 403}: + raise + logger.warning("Vehicle images batch fetch failed: %s", exc) + return 0 + except Exception as exc: + logger.warning("Vehicle images batch fetch failed: %s", exc) + return 0 + + for vehicle_id, car in cars_by_vehicle_id.items(): + images_payload = images_map.get(vehicle_id, []) + fetched_records = self._build_image_records(images_payload) + if not fetched_records: + continue + + before_count = len(car.images) + # Source of truth: /api/vehicles/{id}/images. + car.images = fetched_records + after_count = len(car.images) + if after_count > before_count: + images_added += after_count - before_count + + return images_added + + def _fetch_vehicle_images_batch_with_retry( + self, + client: OpenLaneClient, + vehicle_ids: list[str], + ) -> dict[str, list[dict[str, Any]]]: + retry_schedule = self.openlane_config.retry_schedule_seconds + for attempt in range(len(retry_schedule) + 1): + try: + return client.fetch_vehicle_images_batch(vehicle_ids) + except OpenLaneRequestError as exc: + if exc.status_code in {401, 403}: + raise + is_retryable = exc.status_code == 429 or ( + exc.status_code is not None and exc.status_code >= 500 + ) + if not is_retryable or attempt >= len(retry_schedule): + raise + delay = retry_schedule[attempt] + if exc.status_code == 429: + delay = max(delay * 2, 8.0) + logger.warning( + "Retry vehicle images batch size=%d status=%s delay=%.1fs attempt=%d/%d", + len(vehicle_ids), + exc.status_code, + delay, + attempt + 1, + len(retry_schedule), + ) + time.sleep(delay) + + return {} + + def init_db(self) -> dict[str, Any]: + self.persistence.create_tables() + return {"status": "ok", "message": "DB tables created"} + + def sync_listing( + self, + *, + lane: str | None = None, + limit: int | None = None, + only_new: bool | None = None, + max_pages: int | None = None, + concurrency: int | None = None, + ) -> dict[str, Any]: + """Полный цикл синхронизации: OpenLane API → фильтры → маппинг → DB. + + Параметры берутся из аргументов → runtime_config.json → config.py (в этом порядке). + """ + started_at = time.perf_counter() + rc = self.runtime_config + + # Параметры: аргумент → runtime_config → config default. + lane = lane or rc.sync.lane or "openlane_marketplace" + if limit is None: + limit = rc.sync.limit + if limit is None and self.openlane_config.max_cars_limit > 0: + limit = self.openlane_config.max_cars_limit + if only_new is None: + only_new = rc.sync.only_new + effective_only_new = only_new if only_new is not None else False + max_pages = max_pages or self.openlane_config.max_pages + concurrency = max(1, min(concurrency or self.openlane_config.concurrency, 5)) + + self.persistence.create_tables() + run_id = self.persistence.start_sync_run(lane) + + total_upserted = 0 + total_failed = 0 + total_images = 0 + total_discovered = 0 + total_skipped = 0 + all_origin_ids: set[str] = set() + completed_pages: list[int] = [] + failed_pages: list[int] = [] + status = "success" + error_summary: str | None = None + + # Предзагружаем известные origin_id для раннего пропуска. + known_ids: set[str] = set() + if effective_only_new: + known_ids = self.persistence.get_all_origin_ids_for_lane(prefix="openlane:") + + logger.info( + "sync_listing started: lane=%s max_pages=%s concurrency=%s only_new=%s known=%d", + lane, max_pages, concurrency, effective_only_new, len(known_ids), + ) + + try: + with sync_playwright() as playwright: + browser = self.browser_factory.create_browser(playwright) + try: + contexts = [] + auth_pages = [] + for i in range(concurrency): + storage_state_path = ( + self.openlane_config.storage_state_file + if Path(self.openlane_config.storage_state_file).exists() + else None + ) + ctx = self.browser_factory.create_context(browser, storage_state_path=storage_state_path) + auth_page = self.authenticator.bootstrap_authenticated_context(ctx) + contexts.append(ctx) + auth_pages.append(auth_page) + logger.info("Worker session initialized: worker=%d", i + 1) + + # Последовательная обработка страниц (по одной на контекст). + # Для каждой страницы: fetch → map → upsert. + context_idx = 0 + consecutive_all_known = 0 + consecutive_failures = 0 + early_stop_threshold = 3 # Остановка если 3 страницы подряд без новых записей. + failure_circuit_breaker = 2 # Остановка если 2 подряд ошибки API. + + for page_num in range(1, max_pages + 1): + if limit is not None and total_upserted >= limit: + logger.info("Reached limit=%d, stopping", limit) + break + + if consecutive_failures >= failure_circuit_breaker: + logger.error( + "Circuit breaker: %d consecutive failures — stopping to protect account", + consecutive_failures, + ) + status = "aborted" + break + + ctx = auth_pages[context_idx % len(auth_pages)] + context_idx += 1 + + try: + page_result = self._fetch_page_with_retry(ctx, page_num) + consecutive_failures = 0 # Сброс при успехе. + except Exception as exc: + logger.error("Page %d fetch failed: %s", page_num, exc) + failed_pages.append(page_num) + total_failed += 1 + consecutive_failures += 1 + continue + + if not page_result.records: + logger.info("Page %d returned 0 records — end of listing", page_num) + completed_pages.append(page_num) + break + + # Применяем runtime-фильтры к сырым записям. + filtered_records = apply_filters(page_result.records, rc.filters) + if len(filtered_records) < len(page_result.records): + logger.debug( + "Page %d: %d/%d records passed runtime filters", + page_num, len(filtered_records), len(page_result.records), + ) + + # Маппинг JSON → CarRecord. + car_records = map_openlane_records(filtered_records) + total_discovered += len(page_result.records) + + if not car_records: + logger.warning("Page %d: all %d records failed mapping", page_num, len(page_result.records)) + completed_pages.append(page_num) + continue + + # Фильтрация уже известных записей. + if effective_only_new and known_ids: + new_records = [r for r in car_records if r.origin_id not in known_ids] + skipped = len(car_records) - len(new_records) + total_skipped += skipped + if skipped: + logger.debug("Page %d: skipped %d already known", page_num, skipped) + if not new_records: + consecutive_all_known += 1 + completed_pages.append(page_num) + if consecutive_all_known >= early_stop_threshold: + logger.info( + "Early stop: %d consecutive pages with all known records", + consecutive_all_known, + ) + break + continue + else: + consecutive_all_known = 0 + car_records = new_records + + # Применяем limit. + if limit is not None: + remaining = limit - total_upserted + car_records = car_records[:remaining] + + # Дозапрашиваем фото для машин, у которых их нет в search payload. + try: + added_images = self._enrich_car_records_with_images(ctx, filtered_records, car_records) + if added_images: + logger.debug("Page %d: enriched %d images via vehicle images API", page_num, added_images) + except OpenLaneRequestError as exc: + if exc.status_code in {401, 403}: + raise + logger.warning("Page %d image enrichment skipped: %s", page_num, exc) + + # Upsert батчем. + try: + upsert_result = self.persistence.upsert_cars_batch(car_records) + page_upserted = upsert_result.get("inserted", 0) + upsert_result.get("updated", 0) + page_images = upsert_result.get("images_upserted", 0) + total_upserted += page_upserted + total_images += page_images + for r in car_records: + all_origin_ids.add(r.origin_id) + known_ids.add(r.origin_id) + logger.info( + "Page %d: %d records → %d upserted, %d images", + page_num, len(car_records), page_upserted, page_images, + ) + except Exception as exc: + logger.error("Page %d upsert failed: %s", page_num, exc, exc_info=True) + total_failed += len(car_records) + failed_pages.append(page_num) + continue + + completed_pages.append(page_num) + + finally: + browser.close() + + except Exception as exc: + status = "failed" + error_summary = str(exc)[:500] + logger.error("sync_listing failed: %s", exc, exc_info=True) + + elapsed = time.perf_counter() - started_at + self.persistence.finish_sync_run( + run_id, + status=status, + ids_fetched=total_discovered, + cars_upserted=total_upserted, + cars_failed=total_failed, + images_upserted=total_images, + error_summary=error_summary, + ) + + result = { + "run_id": run_id, + "status": status, + "lane": lane, + "total_discovered": total_discovered, + "cars_upserted": total_upserted, + "cars_failed": total_failed, + "images_upserted": total_images, + "skipped_existing": total_skipped, + "completed_pages": len(completed_pages), + "failed_pages": len(failed_pages), + "elapsed_seconds": round(elapsed, 2), + "full_scan_completed": status == "success", + } + logger.info("sync_listing finished: %s", result) + return result + + def _fetch_page_with_retry(self, authenticated_page, page: int) -> OpenLanePageResult: + client = OpenLaneClient(authenticated_page, self.openlane_config) + retry_schedule = self.openlane_config.retry_schedule_seconds + + for attempt in range(len(retry_schedule) + 1): + try: + return client.fetch_page(page) + except OpenLaneRequestError as exc: + # 403 — возможная блокировка, стоп. + if exc.status_code == 403: + logger.error( + "STOP: page=%d returned 403 Forbidden — aborting to protect account", + page, + ) + raise + # 401 — сессия истекла. + if exc.status_code == 401: + raise + # 429 — rate limit, ретрай с увеличенным backoff. + is_retryable = exc.status_code == 429 or ( + exc.status_code is not None and exc.status_code >= 500 + ) + if not is_retryable or attempt >= len(retry_schedule): + raise + delay = retry_schedule[attempt] + if exc.status_code == 429: + delay = max(delay * 3, 15.0) # 429 → утроенная задержка + logger.warning( + "Retry page=%d status=%s delay=%.1fs attempt=%d/%d", + page, exc.status_code, delay, attempt + 1, len(retry_schedule), + ) + time.sleep(delay) + except Exception: + # Неожиданная ошибка — одна попытка. + if attempt >= min(1, len(retry_schedule)): + raise + delay = retry_schedule[attempt] if attempt < len(retry_schedule) else 5.0 + logger.warning( + "Transient error page=%d delay=%.1fs attempt=%d", + page, delay, attempt + 1, + exc_info=True, + ) + time.sleep(delay) + + raise RuntimeError(f"Page {page} exhausted retries") diff --git a/openlane_scraper/storage/__init__.py b/openlane_scraper/storage/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/openlane_scraper/storage/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/openlane_scraper/storage/db.py b/openlane_scraper/storage/db.py new file mode 100644 index 0000000..a4aa52a --- /dev/null +++ b/openlane_scraper/storage/db.py @@ -0,0 +1,576 @@ +import logging +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Any, Iterator + +from sqlalchemy import create_engine, delete, or_, select, text, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session, sessionmaker + +from ..core.config import Settings +from .models import Base, Car, Image, SyncRun +from .schemas import CarRecord + +logger = logging.getLogger("openlane_scraper.db") + + +CAR_DB_FIELDS = { + col.key for col in Car.__table__.columns + if col.key not in ("id",) +} + +_IN_CHUNK_SIZE = 5000 + + +class PersistenceService: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + engine_kwargs = { + "echo": settings.database.echo, + "future": True, + } + if "postgresql" in settings.database.url: + engine_kwargs["pool_size"] = settings.database.pool_size + engine_kwargs["max_overflow"] = settings.database.max_overflow + engine_kwargs["pool_pre_ping"] = True + engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds + self.engine = create_engine(settings.database.url, **engine_kwargs) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) + + def create_tables(self) -> None: + # В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции. + is_sqlite = self.settings.database.url.startswith("sqlite") + if not is_sqlite and not self.settings.database.auto_create_tables: + return + try: + Base.metadata.create_all(self.engine) + except Exception: + logger.debug("create_tables skipped (schema already exists)") + + @contextmanager + def session_scope(self) -> Iterator[Session]: + session = self.session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def start_sync_run(self, lane: str) -> int: + with self.session_scope() as session: + now = datetime.now(timezone.utc) + stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all() + for stale in stale_runs: + stale.status = "failed" + stale.finished_at = now + if not stale.error_summary: + stale.error_summary = "Recovered stale running sync run before starting a new run" + + run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) + session.add(run) + session.flush() + return int(run.id) + + def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + with self.session_scope() as session: + run = session.get(SyncRun, run_id) + if run is None: + return + run.finished_at = datetime.now(timezone.utc) + run.status = status + run.ids_fetched = ids_fetched + run.cars_upserted = cars_upserted + run.cars_failed = cars_failed + run.images_upserted = images_upserted + run.error_summary = error_summary + + def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]: + if not origin_urls: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: + if not origin_ids: + return set() + with self.session_scope() as session: + rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all() + return {str(row[0]) for row in rows if row and row[0]} + + def get_existing_urls_and_ids( + self, origin_urls: list[str], origin_ids: list[str], + ) -> tuple[set[str], set[str]]: + # Загрузка существующих URL и origin_id. + if not origin_urls and not origin_ids: + return set(), set() + urls: set[str] = set() + ids: set[str] = set() + with self.session_scope() as session: + max_len = max(len(origin_urls), len(origin_ids), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + conditions = [] + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car.origin_url, Car.origin_id).where(or_(*conditions)) + ).all() + for r in rows: + if r[0]: + urls.add(str(r[0])) + if r[1]: + ids.add(str(r[1])) + return urls, ids + + @staticmethod + def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: + if not images: + return + session.add_all([ + Image( + fullres_image=str(img["fullres_image"]), + preview_image=str(img["preview_image"]), + order_index=int(img.get("order_index", 0)), + car_id=car_id, + ) + for img in images + ]) + + @staticmethod + def _car_payload(record: CarRecord) -> dict[str, object]: + payload = record.model_dump(mode="python") + return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + + def _is_postgres(self) -> bool: + return self.engine.dialect.name == "postgresql" + + def _load_existing_cars( + self, + session: Session, + origin_ids: list[str], + origin_urls: list[str], + ) -> tuple[dict[str, Car], dict[str, Car]]: + existing_by_id: dict[str, Car] = {} + existing_by_url: dict[str, Car] = {} + if not origin_ids and not origin_urls: + return existing_by_id, existing_by_url + + existing_cars: list[Car] = [] + max_len = max(len(origin_ids), len(origin_urls), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + conditions = [] + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car).where(or_(*conditions)) + ).scalars().all() + existing_cars.extend(rows) + + for car in existing_cars: + if car.origin_id: + existing_by_id[car.origin_id] = car + if car.origin_url: + existing_by_url[car.origin_url] = car + return existing_by_id, existing_by_url + + def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]: + existing_images_map: dict[int, set[str]] = {} + if not car_ids: + return existing_images_map + + car_id_list = list(car_ids) + for i in range(0, len(car_id_list), _IN_CHUNK_SIZE): + chunk = car_id_list[i:i + _IN_CHUNK_SIZE] + img_rows = session.execute( + select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk)) + ).all() + for cid, furl in img_rows: + existing_images_map.setdefault(int(cid), set()).add(str(furl)) + return existing_images_map + + @staticmethod + def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]: + return { + key: getattr(insert_stmt.excluded, key) + for key in CAR_DB_FIELDS + } + + def _replace_images_for_car( + self, + session: Session, + car_id: int, + images: list[dict[str, object]], + origin_id: str, + ) -> int: + nested = session.begin_nested() + try: + session.execute(delete(Image).where(Image.car_id == car_id)) + self._add_images(session, car_id, images) + session.flush() + nested.commit() + return len(images) + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True) + return 0 + + def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]: + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + entries: list[dict[str, object]] = [] + upsert_payloads: list[dict[str, object]] = [] + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + car_by_id = existing_by_id.get(record.origin_id) + car_by_url = existing_by_url.get(record.origin_url) + entry: dict[str, object] = {"record": record, "images": images, "car_id": None} + + if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + entry["car_id"] = int(car_by_url.id) + updated += 1 + else: + upsert_payloads.append(payload) + if car_by_id is not None: + updated += 1 + else: + inserted += 1 + entries.append(entry) + + session.flush() + + if upsert_payloads: + insert_stmt = pg_insert(Car).values(upsert_payloads) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id, Car.origin_id) + rows = session.execute(upsert_stmt).all() + car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows} + for entry in entries: + if entry["car_id"] is not None: + continue + record = entry["record"] + car_id = car_ids_by_origin_id.get(record.origin_id) + if car_id is None: + raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}") + entry["car_id"] = car_id + + car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None} + existing_images_map = self._load_existing_image_urls(session, car_ids) + images_by_car_id: dict[int, list[dict[str, object]]] = {} + replace_ids: list[int] = [] + + for entry in entries: + car_id = int(entry["car_id"]) + images = entry["images"] + new_image_urls = { + str(img.get("fullres_image", "")) + for img in images + if img.get("fullres_image") + } + old_image_urls = existing_images_map.get(car_id, set()) + if new_image_urls != old_image_urls: + replace_ids.append(car_id) + images_by_car_id[car_id] = images + else: + images_total += len(old_image_urls) + + if replace_ids: + for i in range(0, len(replace_ids), _IN_CHUNK_SIZE): + chunk = replace_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car_id in replace_ids: + images = images_by_car_id[car_id] + self._add_images(session, car_id, images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def upsert_car(self, record: CarRecord): + # Вставка или обновление автомобиля по origin_id/origin_url. + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + with self.session_scope() as session: + if self._is_postgres(): + car_by_url = session.execute( + select(Car).where(Car.origin_url == record.origin_url) + ).scalar_one_or_none() + if car_by_url is not None and car_by_url.origin_id != record.origin_id: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + session.flush() + car_id = int(car_by_url.id) + action = "updated" + else: + existed = session.execute( + select(Car.id).where(Car.origin_id == record.origin_id) + ).scalar_one_or_none() is not None + insert_stmt = pg_insert(Car).values(**payload) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id) + car_id = int(session.execute(upsert_stmt).scalar_one()) + action = "updated" if existed or car_by_url is not None else "inserted" + + images_upserted = self._replace_images_for_car( + session, car_id, images, record.origin_id, + ) + return {"car_id": car_id, "images_upserted": images_upserted, "action": action} + + car = session.execute( + select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) + ).scalar_one_or_none() + action = "inserted" + if car is None: + car = Car(**payload) + session.add(car) + session.flush() + else: + action = "updated" + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + session.flush() + images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id) + return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action} + self._add_images(session, int(car.id), images) + session.flush() + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]: + """Пакетный upsert нескольких автомобилей в одной транзакции. + + Оптимизации: + - Дедупликация записей по origin_id перед вставкой. + - Chunked IN-queries для больших списков (обход лимита PG параметров). + - Пропуск перезаписи изображений, если набор URL не изменился. + - Один DELETE по car_id IN (...) вместо удаления по одному. + - Fallback на по-одному upsert если batch commit упал. + """ + # ── Дедупликация записей внутри батча ── + seen_ids: dict[str, int] = {} + unique_records: list[CarRecord] = [] + for idx, r in enumerate(records): + key = r.origin_id or r.origin_url + if key in seen_ids: + logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key]) + continue + seen_ids[key] = idx + unique_records.append(r) + + if len(unique_records) < len(records): + logger.info("Deduped batch: %d → %d records", len(records), len(unique_records)) + records = unique_records + + try: + return self._upsert_cars_batch_inner(records) + except Exception as exc: + logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc) + return self._upsert_cars_individually(records) + + def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]: + """Внутренняя реализация batched upsert (одна транзакция).""" + if self._is_postgres(): + return self._upsert_cars_batch_postgres(records) + + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + # Получаем существующие записи chunked-запросами. + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + # Предзагружаем ВСЕ изображения для обновляемых машин одним запросом. + existing_car_ids = set() + for record in records: + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is not None: + existing_car_ids.add(int(car.id)) + + # Строим маппинг car_id → set(image_urls) для сравнения. + existing_images_map = self._load_existing_image_urls(session, existing_car_ids) + + new_cars: list[tuple[Car, list[dict]]] = [] + update_cars_needing_images: list[tuple[Car, list[dict]]] = [] + + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is None: + car = Car(**payload) + session.add(car) + inserted += 1 + new_cars.append((car, images)) + else: + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + updated += 1 + + # Проверяем, изменились ли изображения. + new_image_urls = {img.get("fullres_image", "") for img in images} + old_image_urls = existing_images_map.get(int(car.id), set()) + if new_image_urls != old_image_urls: + update_cars_needing_images.append((car, images)) + else: + images_total += len(old_image_urls) + + # Один flush для всех вставок. + session.flush() + + # Добавляем изображения для новых автомобилей. + for car, images in new_cars: + self._add_images(session, int(car.id), images) + images_total += len(images) + + # Массово обновляем изображения только для машин с изменёнными картинками. + if update_cars_needing_images: + update_ids = [int(car.id) for car, _ in update_cars_needing_images] + for i in range(0, len(update_ids), _IN_CHUNK_SIZE): + chunk = update_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car, images in update_cars_needing_images: + self._add_images(session, int(car.id), images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]: + # Fallback на поштучный upsert. + inserted = 0 + updated = 0 + images_total = 0 + for record in records: + try: + result = self.upsert_car(record) + action = result.get("action", "inserted") + if action == "inserted": + inserted += 1 + else: + updated += 1 + images_total += int(result.get("images_upserted", 0)) + except Exception as exc: + logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "openlane") -> int: + """Помечает авто как проданные, если их нет в активном листинге (по origin_id).""" + if not active_origin_ids: + return 0 + with self.session_scope() as session: + stmt = ( + update(Car) + .where(Car.origin_id.notin_(active_origin_ids)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("openlane:%")) + .values(is_sold=True) + ) + result = session.execute(stmt) + count = result.rowcount or 0 + if count: + logger.info("Marked %d cars as sold (no longer in listing)", count) + return count + + def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "openlane") -> int: + """Помечает авто как проданные, если их URL нет в активном листинге. + + Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, + что кардинально быстрее при больших объёмах (100K+ URLs). + """ + if not active_origin_urls: + return 0 + + is_postgres = "postgresql" in self.settings.database.url + + with self.session_scope() as session: + if is_postgres: + # Создаём временную таблицу с активными URL. + session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP")) + session.execute(text("TRUNCATE _active_urls")) + + # Вставляем активные URL чанками. + url_list = list(active_origin_urls) + for i in range(0, len(url_list), _IN_CHUNK_SIZE): + chunk = url_list[i:i + _IN_CHUNK_SIZE] + values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) + params = {f"u{j}": url for j, url in enumerate(chunk)} + session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params) + + # Создаём индекс на временной таблице для ускорения JOIN. + session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)")) + + # Массовая пометка проданных в PostgreSQL. + result = session.execute(text(""" + UPDATE cars + SET is_sold = TRUE + FROM ( + SELECT c.id + FROM cars c + LEFT JOIN _active_urls a ON c.origin_url = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'openlane:%%' + ) sub + WHERE cars.id = sub.id + """)) + count = result.rowcount or 0 + else: + # Упрощённый путь для SQLite. + stmt = ( + update(Car) + .where(Car.origin_url.notin_(active_origin_urls)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("openlane:%")) + .values(is_sold=True) + ) + result = session.execute(stmt) + count = result.rowcount or 0 + + if count: + logger.info("Marked %d cars as sold by URL (no longer in listing)", count) + return count + + def get_all_origin_ids_for_lane(self, prefix: str = "openlane:") -> set[str]: + """Возвращает все известные origin_id для заданного префикса. + + Использует yield_per для потоковой загрузки при большом количестве записей. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} \ No newline at end of file diff --git a/openlane_scraper/storage/enums.py b/openlane_scraper/storage/enums.py new file mode 100644 index 0000000..253c73f --- /dev/null +++ b/openlane_scraper/storage/enums.py @@ -0,0 +1,25 @@ +CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") +DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA") +GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") +STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA") +BODY_TYPE_ENUM_VALUES = ( + "COUPE", + "SUV", + "HATCHBACK", + "MINIVAN", + "SEDAN", + "STATION_WAGON", + "PICKUP", + "TRUCK", + "OPEN", + "RV", + "OTHER", + "NA", +) +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") +ORIGIN_ENUM_VALUES = ( + "OPENLANE", + "IAAI", + "NA", +) +SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA") diff --git a/openlane_scraper/storage/models.py b/openlane_scraper/storage/models.py new file mode 100644 index 0000000..29d5566 --- /dev/null +++ b/openlane_scraper/storage/models.py @@ -0,0 +1,82 @@ +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from .enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) + + +class Base(DeclarativeBase): + pass + + +class Car(Base): + __tablename__ = "cars" + __table_args__ = ( + Index("ix_cars_brand_model", "brand", "model"), + Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True) + model: Mapped[str] = mapped_column(String(50), nullable=False) + year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) + price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD") + mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + color: Mapped[str] = mapped_column(String(), nullable=False, default="other") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER") + engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) + is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) + non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + slug: Mapped[str] = mapped_column(String(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) + images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") + + +class Image(Base): + __tablename__ = "images" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + fullres_image: Mapped[str] = mapped_column(String(), nullable=False) + preview_image: Mapped[str] = mapped_column(String(), nullable=False) + order_index: Mapped[int] = mapped_column(Integer, nullable=False) + car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True) + car: Mapped[Car] = relationship("Car", back_populates="images") + + +class SyncRun(Base): + __tablename__ = "sync_runs" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False, index=True) + lane: Mapped[str] = mapped_column(Text, nullable=False) + ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/openlane_scraper/storage/schemas.py b/openlane_scraper/storage/schemas.py new file mode 100644 index 0000000..7af7c43 --- /dev/null +++ b/openlane_scraper/storage/schemas.py @@ -0,0 +1,87 @@ +from datetime import datetime, timezone +from pydantic import BaseModel, ConfigDict, Field + + +class ImageRecord(BaseModel): + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRecord(BaseModel): + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str + origin_id: str + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str + last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + images: list[ImageRecord] = Field(default_factory=list) + + +class ImageRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str = "" + origin_id: str = "" + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str = "" + last_seen_at: datetime | None = None + images: list[ImageRead] = Field(default_factory=list) + diff --git a/openlane_scraper/worker/__init__.py b/openlane_scraper/worker/__init__.py new file mode 100644 index 0000000..841be13 --- /dev/null +++ b/openlane_scraper/worker/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/openlane_scraper/worker/celery_app.py b/openlane_scraper/worker/celery_app.py new file mode 100644 index 0000000..f6d227a --- /dev/null +++ b/openlane_scraper/worker/celery_app.py @@ -0,0 +1,124 @@ +# Инициализация Celery-приложения и периодических задач. + +import logging + +from celery import Celery +from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging +from redis import Redis + +from ..core.config import settings +from ..core.logs import setup_logging + +logger = logging.getLogger("openlane_scraper.worker.celery_app") +STARTUP_SYNC_DISPATCH_KEY = "openlane:state:startup_sync_dispatched" + + +@celery_setup_logging.connect +def _configure_logging(loglevel=None, **kwargs): + # Перехватываем логирование Celery и пишем только в stderr (Docker logs). + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +@worker_process_init.connect +def _on_worker_process_init(**kwargs): + # Повторно настраиваем логирование в каждом дочернем prefork-процессе, + # чтобы StreamHandler(stderr) корректно работал после fork. + level = settings.log_level if settings.log_level else "INFO" + setup_logging(level, None) + + +def _broker_url() -> str: + return settings.celery.broker_url or settings.redis.url + + +def _result_backend() -> str: + return settings.celery.result_backend or settings.redis.url + + +celery_app = Celery( + "openlane_scraper", + broker=_broker_url(), + backend=_result_backend(), +) + +# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120), +# ограничиваем, чтобы зависший worker не жил вечно. +_soft = settings.celery.task_soft_time_limit +_hard = settings.celery.task_time_limit +_max_hard = _soft + 120 if _soft else _hard +if _hard > _max_hard: + logger.warning( + "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; " + "clamping hard limit to %d", + _hard, _soft, _max_hard, + ) + _hard = _max_hard + +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_soft_time_limit=_soft, + task_time_limit=_hard, + task_acks_late=True, + task_reject_on_worker_lost=True, + task_track_started=True, + worker_concurrency=settings.celery.worker_concurrency, + worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child, + worker_pool="prefork", + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + broker_transport_options={ + "visibility_timeout": settings.celery.broker_visibility_timeout, + }, + result_expires=86400, + worker_redirect_stdouts=False, + worker_hijack_root_logger=False, + beat_schedule={ + "periodic-sync-listing": { + "task": "openlane_scraper.worker.tasks.sync_listing_task", + "schedule": settings.celery.beat_sync_interval_minutes * 60.0, + "args": (), + "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False}, + "options": {"queue": "scraping"}, + } + }, + task_routes={ + "openlane_scraper.worker.tasks.*": {"queue": "scraping"} + }, +) + +celery_app.autodiscover_tasks(["openlane_scraper.worker"]) + + +@worker_ready.connect +def _on_worker_ready(**kwargs): + """Сразу при старте worker отправляем первую задачу sync_listing, + чтобы не ждать час до первого beat-цикла.""" + try: + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) + except Exception: + logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True) + return + + if not should_dispatch: + logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue") + return + + logger.info("Worker ready — dispatching initial sync_listing task") + celery_app.send_task( + "openlane_scraper.worker.tasks.sync_listing_task", + kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False}, + queue="scraping", + ) \ No newline at end of file diff --git a/openlane_scraper/worker/tasks.py b/openlane_scraper/worker/tasks.py new file mode 100644 index 0000000..6e62a05 --- /dev/null +++ b/openlane_scraper/worker/tasks.py @@ -0,0 +1,360 @@ +# Задачи Celery для синхронизации OpenLane marketplace → DB. + +import logging +import random +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from threading import Event, Thread + +from billiard.exceptions import SoftTimeLimitExceeded +from celery import shared_task +from redis import Redis + +from ..core.config import Settings +from ..scraper import OpenLaneScraper +from ..storage.db import PersistenceService + +logger = logging.getLogger("openlane_scraper.worker.tasks") + +SYNC_LISTING_LOCK_KEY = "openlane:locks:sync_listing" +SYNC_FULL_SCAN_DONE_KEY = "openlane:state:sync_full_scan_done" +SYNC_LISTING_TASK_NAME = "openlane_scraper.worker.tasks.sync_listing_task" + + +def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): + last_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + return func() + except Exception as exc: + last_exc = exc + if attempt >= attempts: + break + delay = base_delay_s * (2 ** (attempt - 1)) + logger.warning( + "Operation failed (attempt %d/%d): %s. Retrying in %.1fs", + attempt, attempts, exc, delay, + ) + time.sleep(delay) + if last_exc is not None: + raise last_exc + + +def _run_browser_job(func, *args, **kwargs): + settings = Settings() + soft = settings.celery.task_soft_time_limit + hard = settings.celery.task_time_limit + wait_timeout = min(hard, soft + 120) if soft and hard else None + + executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="openlane-browser") + future = executor.submit(func, *args, **kwargs) + try: + result = future.result(timeout=wait_timeout) + except SoftTimeLimitExceeded: + future.cancel() + executor.shutdown(wait=False, cancel_futures=True) + raise + except TimeoutError: + future.cancel() + executor.shutdown(wait=False, cancel_futures=True) + raise SoftTimeLimitExceeded("Browser thread did not finish within time limit") + except Exception: + executor.shutdown(wait=False, cancel_futures=True) + raise + else: + executor.shutdown(wait=True) + return result + + +def _sync_listing_lock_ttl_seconds() -> int: + settings = Settings() + soft = settings.celery.task_soft_time_limit + hard = settings.celery.task_time_limit + effective_hard = min(hard, soft + 120) if soft else hard + return max(effective_hard + 120, 300) + + +def _get_persistence() -> PersistenceService: + settings = Settings() + persistence = PersistenceService(settings) + + def _ping_db() -> None: + with persistence.engine.connect() as conn: + conn.exec_driver_sql("SELECT 1") + + _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0) + return persistence + + +def _get_redis() -> Redis: + settings = Settings() + redis_client = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, + socket_timeout=settings.redis.socket_timeout_seconds, + health_check_interval=settings.redis.health_check_interval_seconds, + retry_on_timeout=True, + ) + + def _ping_redis() -> None: + redis_client.ping() + + _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0) + return redis_client + + +def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: + try: + acquired = bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + if acquired: + return True + ttl = redis_client.ttl(key) + if ttl is not None and ttl < 0: + logger.warning("Detected stale lock without TTL, removing: %s", key) + redis_client.delete(key) + return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + return False + except Exception: + logger.warning("Failed to acquire lock %s", key, exc_info=True) + return False + + +def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: + try: + refreshed = redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) + end + return 0 + """, + 1, key, owner_token, int(ttl_seconds), + ) + return bool(refreshed) + except Exception: + logger.warning("Failed to refresh lock %s", key, exc_info=True) + return None + + +def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: + try: + redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) + end + return 0 + """, + 1, key, owner_token, + ) + except Exception: + logger.warning("Failed to release lock %s", key, exc_info=True) + + +def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool: + try: + inspector = celery_app.control.inspect(timeout=1.0) + snapshots = [ + inspector.active() or {}, + inspector.reserved() or {}, + inspector.scheduled() or {}, + ] + except Exception: + logger.warning("Failed to inspect Celery workers for running sync tasks", exc_info=True) + return True + + for snapshot in snapshots: + for entries in snapshot.values(): + for entry in entries or []: + task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") + if task_name != SYNC_LISTING_TASK_NAME: + continue + entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "") + if exclude_task_id and entry_id == exclude_task_id: + continue + return True + return False + + +def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool: + try: + owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY) + if not owner_token: + return False + except Exception: + logger.warning("Failed to read sync listing lock before cleanup", exc_info=True) + return False + + if _has_running_sync_listing_tasks(celery_app, exclude_task_id=current_task_id): + logger.info("sync_listing lock preserved: active task still detected") + return False + + try: + ttl = redis_client.ttl(SYNC_LISTING_LOCK_KEY) + redis_client.delete(SYNC_LISTING_LOCK_KEY) + logger.warning( + "Removed orphan sync_listing lock owner=%s ttl=%s after worker restart", + owner_token, ttl, + ) + return True + except Exception: + logger.warning("Failed to clear orphan sync listing lock", exc_info=True) + return False + + +def _is_full_scan_done(redis_client: Redis) -> bool: + try: + value = redis_client.get(SYNC_FULL_SCAN_DONE_KEY) + except Exception: + logger.warning("Failed to read full scan state", exc_info=True) + return False + return str(value or "").strip() == "1" + + +def _set_full_scan_done(redis_client: Redis, done: bool) -> None: + try: + redis_client.set(SYNC_FULL_SCAN_DONE_KEY, "1" if done else "0") + except Exception: + logger.warning("Failed to persist full scan state", exc_info=True) + + +def _start_lock_heartbeat( + redis_client: Redis, key: str, owner_token: str, ttl_seconds: int, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) + + def _heartbeat() -> None: + while not stop_event.wait(interval_seconds): + refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) + if refreshed is False: + logger.warning("Lost sync_listing lock ownership for %s", owner_token) + return + + thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) + thread.start() + return stop_event, thread + + +@shared_task( + name="openlane_scraper.worker.tasks.sync_listing_task", + bind=True, + max_retries=3, + default_retry_delay=120, + acks_late=True, +) +def sync_listing_task( + self, + lane: str = "openlane_marketplace", + limit: int | None = None, + only_new: bool | None = None, + max_pages: int | None = None, + concurrency: int | None = None, +): + """Полный цикл синхронизации OpenLane marketplace → DB.""" + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + owner_token = f"{task_id}:{uuid.uuid4().hex}" + redis_client = _get_redis() + + lock_acquired = False + lock_ttl = _sync_listing_lock_ttl_seconds() + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + orphan_cleared = _clear_orphan_sync_listing_lock(redis_client, self.app, current_task_id=task_id) + if orphan_cleared: + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + logger.info("sync_listing_task skipped: another sync is already running") + return { + "status": "skipped", + "reason": "sync_already_running", + "task_id": task_id, + } + + try: + full_scan_done = _is_full_scan_done(redis_client) + effective_only_new = False if not full_scan_done else only_new + + if not full_scan_done: + logger.info("Bootstrap mode: forcing full scan (only_new=False) until first complete run") + + # Рандомный jitter (0–120s) перед стартом. + jitter = random.uniform(0, 120) + logger.info("Anti-pattern jitter: waiting %.0fs before starting scrape", jitter) + time.sleep(jitter) + + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl, + ) + self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) + + def _job(): + with OpenLaneScraper() as scraper: + return scraper.sync_listing( + lane=lane, + limit=limit, + only_new=effective_only_new, + max_pages=max_pages, + concurrency=concurrency, + ) + + result = _run_browser_job(_job) + + if not full_scan_done: + if result.get("status") == "success": + _set_full_scan_done(redis_client, True) + logger.info("Bootstrap full scan completed; hourly schedule continues") + else: + _set_full_scan_done(redis_client, False) + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + } + logger.info( + "sync_listing_task completed: status=%s, %d upserted, %d failed", + summary["status"], summary["cars_upserted"], summary["cars_failed"], + ) + return summary + + except SoftTimeLimitExceeded: + logger.warning("sync_listing_task soft timeout exceeded — partial progress already saved to DB") + return { + "status": "timed_out", + "task_id": task_id, + "reason": "soft_time_limit_exceeded", + } + + except Exception as exc: + logger.error("sync_listing_task failed: %s", exc, exc_info=True) + try: + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + logger.error("sync_listing_task max retries exceeded, giving up") + return { + "status": "failed", + "task_id": task_id, + "error": str(exc), + } + finally: + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) + if lock_acquired: + _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token)