diff --git a/.env b/.env deleted file mode 100644 index ce978c2..0000000 --- a/.env +++ /dev/null @@ -1,81 +0,0 @@ -IAAI_HEADLESS=true -IAAI_LOG_LEVEL=INFO -# IAAI_LOG_FILE=iaai_scraper.log - -# Network capture settings. -IAAI_CAPTURE_SAME_ORIGIN_ONLY=true -IAAI_MAX_CAPTURED_REQUESTS=40 -IAAI_MAX_CAPTURED_JSON_RESPONSES=30 - -# Sequential listing-first mode. -IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars -IAAI_MAX_PAGES_PER_RUN=10 -IAAI_MAX_VEHICLES_PER_RUN=30 -IAAI_PAGE_LINK_LIMIT=100 -IAAI_INCLUDE_PAGINATION=true -IAAI_COLLECT_CURRENT_PAGE_ONLY=false - -# Human-like pacing. -IAAI_HUMAN_PACE_ENABLED=true -IAAI_AFTER_LISTING_OPEN_MIN_S=2.5 -IAAI_AFTER_LISTING_OPEN_MAX_S=4.5 -IAAI_AFTER_FILTER_ACTION_MIN_S=2.0 -IAAI_AFTER_FILTER_ACTION_MAX_S=4.0 -IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5 -IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0 -IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0 -IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5 -IAAI_BETWEEN_VEHICLES_MIN_S=3.0 -IAAI_BETWEEN_VEHICLES_MAX_S=6.0 -IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0 -IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0 - -# Sync settings -IAAI_SYNC_ONLY_NEW=true -IAAI_TOKENS_FILE=tokens.json -IAAI_RUNTIME_CONFIG_FILE=runtime_config.json - -# Retry / backoff -IAAI_RETRY_DELAY_SECONDS=2.5 -IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 -IAAI_RETRY_JITTER_SECONDS=0.25 - -# Session persistence (cookies) -IAAI_STORAGE_STATE_PATH=storage_state.json -IAAI_SESSION_MAX_AGE_DAYS=30 -IAAI_SESSION_SAVE_ON_EXIT=true - -# IAAI login credentials -IAAI_LOGIN_EMAIL=ofcmkrtzhr@hotmail.com -IAAI_LOGIN_PASSWORD=FMGX3Q58syz8 - -# Proxy (HTTP/HTTPS preferred for Playwright) -# Chromium does not support SOCKS5 proxy authentication directly. -# Use residential or mobile USA proxy. -# IAAI_PROXY_SERVER=http://proxy.example.com:8080 -# IAAI_PROXY_USERNAME= -# IAAI_PROXY_PASSWORD= - -# Database — для локального CLI без Docker раскомментируйте SQLite: -# IAAI_DATABASE_URL=sqlite:///iaai_scraper.db -# Для Docker используется PostgreSQL из docker-compose.yml автоматически. -# Для локального PostgreSQL: IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper -IAAI_DATABASE_ECHO=false -IAAI_DATABASE_POOL_SIZE=5 -IAAI_DATABASE_MAX_OVERFLOW=10 -IAAI_DATABASE_AUTO_CREATE_TABLES=true - -# Redis (Celery broker) не нужен для CLI-режима. -# Раскомментировать для запуска API + Worker + Beat. -# IAAI_REDIS_URL=redis://localhost:6379/0 - -# Celery —не нужен для CLI-режима. -# CELERY_BROKER_URL=redis://localhost:6379/0 -# CELERY_RESULT_BACKEND=redis://localhost:6379/0 -# CELERY_TASK_SOFT_TIME_LIMIT=600 -# CELERY_TASK_TIME_LIMIT=900 -# CELERY_WORKER_CONCURRENCY=1 -# CELERY_WORKER_MAX_TASKS_PER_CHILD=20 -# CELERY_BROKER_VISIBILITY_TIMEOUT=7200 -# CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 -# CELERY_BEAT_SYNC_LIMIT=26 diff --git a/.gitignore b/.gitignore index 86adc10..682bbf8 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ dist/ build/ artifacts/ celerybeat-schedule* -tokens_data/ \ No newline at end of file +tokens_data/ +.env \ No newline at end of file diff --git a/iaai_scraper/__init__.py b/iaai_scraper/__init__.py deleted file mode 100644 index c9c2ef6..0000000 --- a/iaai_scraper/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__: list[str] = [] diff --git a/iaai_scraper/api/__init__.py b/iaai_scraper/api/__init__.py deleted file mode 100644 index b94a1e8..0000000 --- a/iaai_scraper/api/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .app import create_app - -__all__ = ["create_app"] diff --git a/iaai_scraper/api/app.py b/iaai_scraper/api/app.py deleted file mode 100644 index ba73b00..0000000 --- a/iaai_scraper/api/app.py +++ /dev/null @@ -1,40 +0,0 @@ -# Создание FastAPI-приложения и настройка его жизненного цикла. - -from contextlib import asynccontextmanager - -from fastapi import FastAPI - -from ..core.config import Settings -from ..storage.db import PersistenceService -from .routes import cars, health, tasks - - -@asynccontextmanager -async def lifespan(app: FastAPI): - # Жизненный цикл. - # Таблицы создаются через Alembic-миграции (сервис migrate). - yield - - -def create_app(settings: Settings | None = None) -> FastAPI: - _settings = settings or Settings() - - app = FastAPI( - title="IAAI Scraper API", - description="REST API для управления задачами скрапинга IAAI и просмотра данных", - version="1.0.0", - lifespan=lifespan, - ) - - app.state.settings = _settings - app.state.persistence = PersistenceService(_settings) - - app.include_router(health.router, tags=["health"]) - app.include_router(cars.router, prefix="/api/v1", tags=["cars"]) - app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"]) - - return app - - -# Экземпляр приложения для запуска через uvicorn. -app = create_app() diff --git a/iaai_scraper/api/deps.py b/iaai_scraper/api/deps.py deleted file mode 100644 index 86b4bc3..0000000 --- a/iaai_scraper/api/deps.py +++ /dev/null @@ -1,9 +0,0 @@ -# Вспомогательные зависимости для FastAPI-роутов. - -from fastapi import Request - -from ..storage.db import PersistenceService - - -def get_persistence(request: Request) -> PersistenceService: - return request.app.state.persistence diff --git a/iaai_scraper/api/routes/__init__.py b/iaai_scraper/api/routes/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/iaai_scraper/api/routes/cars.py b/iaai_scraper/api/routes/cars.py deleted file mode 100644 index 7195d48..0000000 --- a/iaai_scraper/api/routes/cars.py +++ /dev/null @@ -1,103 +0,0 @@ -# Роуты для просмотра автомобилей и агрегированной статистики. - -from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import func, select - -from ..deps import get_persistence -from ...storage.db import PersistenceService -from ...storage.models import Car, Image -from ...storage.schemas import CarRead - -router = APIRouter() - - -@router.get("/cars") -def list_cars( - page: int = Query(1, ge=1), - per_page: int = Query(20, ge=1, le=100), - brand: str | None = None, - model: str | None = None, - year_min: int | None = None, - year_max: int | None = None, - is_sold: bool | None = None, - persistence: PersistenceService = Depends(get_persistence), -): - # Список автомобилей с пагинацией и фильтрами - with persistence.session_scope() as session: - query = select(Car) - - if brand: - query = query.where(Car.brand.ilike(f"%{brand}%")) - if model: - query = query.where(Car.model.ilike(f"%{model}%")) - if year_min is not None: - query = query.where(Car.year >= year_min) - if year_max is not None: - query = query.where(Car.year <= year_max) - if is_sold is not None: - query = query.where(Car.is_sold == is_sold) - - count_query = select(func.count()).select_from(query.subquery()) - total = session.execute(count_query).scalar() or 0 - - offset = (page - 1) * per_page - cars = session.execute( - query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page) - ).scalars().all() - - return { - "total": total, - "page": page, - "per_page": per_page, - "pages": (total + per_page - 1) // per_page if per_page else 0, - "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars], - } - - -@router.get("/cars/{car_id}") -def get_car( - car_id: int, - persistence: PersistenceService = Depends(get_persistence), -): - # Детальная информация об автомобиле с изображениями - with persistence.session_scope() as session: - car = session.get(Car, car_id) - if car is None: - raise HTTPException(status_code=404, detail="Car not found") - return CarRead.model_validate(car).model_dump(mode="json") - - -@router.get("/cars/by-origin/{origin_id}") -def get_car_by_origin( - origin_id: str, - persistence: PersistenceService = Depends(get_persistence), -): - # Поиск автомобиля по origin_id - with persistence.session_scope() as session: - car = session.execute( - select(Car).where(Car.origin_id == origin_id) - ).scalars().first() - if car is None: - raise HTTPException(status_code=404, detail="Car not found") - return CarRead.model_validate(car).model_dump(mode="json") - - -@router.get("/stats") -def get_stats(persistence: PersistenceService = Depends(get_persistence)): - # Общая статистика по БД - with persistence.session_scope() as session: - total_cars = session.execute(select(func.count(Car.id))).scalar() or 0 - total_images = session.execute(select(func.count(Image.id))).scalar() or 0 - brands = session.execute( - select(Car.brand, func.count(Car.id)) - .group_by(Car.brand) - .order_by(func.count(Car.id).desc()) - .limit(20) - ).all() - - return { - "total_cars": total_cars, - "total_images": total_images, - "top_brands": [{"brand": b, "count": c} for b, c in brands], - } - diff --git a/iaai_scraper/api/routes/health.py b/iaai_scraper/api/routes/health.py deleted file mode 100644 index a7de8d7..0000000 --- a/iaai_scraper/api/routes/health.py +++ /dev/null @@ -1,30 +0,0 @@ -# Роут проверки доступности сервиса и соединения с БД. - -import logging - -from fastapi import APIRouter, Depends -from sqlalchemy import text - -from ..deps import get_persistence -from ...storage.db import PersistenceService - -router = APIRouter() -logger = logging.getLogger("iaai_scraper.api.health") - - -@router.get("/health") -def health_check(persistence: PersistenceService = Depends(get_persistence)): - # Проверка API и БД - db_ok = False - try: - with persistence.session_scope() as session: - session.execute(text("SELECT 1")) - db_ok = True - except Exception: - logger.warning("Health DB check failed", exc_info=True) - - return { - "status": "ok" if db_ok else "degraded", - "service": "iaai-scraper-api", - "database": "connected" if db_ok else "unavailable", - } diff --git a/iaai_scraper/api/routes/tasks.py b/iaai_scraper/api/routes/tasks.py deleted file mode 100644 index 3979d34..0000000 --- a/iaai_scraper/api/routes/tasks.py +++ /dev/null @@ -1,120 +0,0 @@ -# Роуты запуска задач синхронизации и просмотра истории sync-runs. - -from fastapi import APIRouter, Depends, Query -from pydantic import BaseModel -from sqlalchemy import select, func - -from ..deps import get_persistence -from ...storage.db import PersistenceService -from ...storage.models import SyncRun -from ...worker.celery_app import celery_app -from ...worker.tasks import sync_vehicle_task, sync_listing_task - -router = APIRouter() - - -class SyncVehicleRequest(BaseModel): - vehicle_url: str - lane: str = "iaai" - - -class SyncListingRequest(BaseModel): - make: str | None = None - model: str | None = None - lane: str = "iaai_cars" - limit: int | None = None - only_new: bool | None = None - - -@router.post("/tasks/sync-vehicle") -def start_sync_vehicle( - body: SyncVehicleRequest, -): - # Запустить задачу скрапинга одного автомобиля через Celery - result = sync_vehicle_task.apply_async( - kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, - queue="scraping", - ) - - return { - "task_id": result.id, - "status": "queued", - "vehicle_url": body.vehicle_url, - } - - -@router.post("/tasks/sync-listing") -def start_sync_listing( - body: SyncListingRequest, -): - # Запустить задачу полного цикла листинга через Celery - result = sync_listing_task.apply_async( - kwargs={ - "make": body.make, - "model": body.model, - "lane": body.lane, - "limit": body.limit, - "only_new": body.only_new, - }, - queue="scraping", - ) - - return { - "task_id": result.id, - "status": "queued", - } - - -@router.get("/tasks/{task_id}") -def get_task_status(task_id: str): - result = celery_app.AsyncResult(task_id) - - payload: dict = { - "task_id": task_id, - "state": result.state, - } - - if result.successful(): - payload["result"] = result.result - elif result.failed(): - payload["error"] = str(result.result) - elif result.info is not None: - payload["meta"] = result.info - - return payload - - -@router.get("/sync-runs") -def list_sync_runs( - page: int = Query(1, ge=1), - per_page: int = Query(20, ge=1, le=100), - persistence: PersistenceService = Depends(get_persistence), -): - # История запусков синхронизации - with persistence.session_scope() as session: - total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 - offset = (page - 1) * per_page - runs = session.execute( - select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) - ).scalars().all() - - return { - "total": total, - "page": page, - "per_page": per_page, - "items": [ - { - "id": r.id, - "started_at": r.started_at.isoformat() if r.started_at else None, - "finished_at": r.finished_at.isoformat() if r.finished_at else None, - "status": r.status, - "lane": r.lane, - "ids_fetched": r.ids_fetched, - "cars_upserted": r.cars_upserted, - "cars_failed": r.cars_failed, - "images_upserted": r.images_upserted, - "error_summary": r.error_summary, - } - for r in runs - ], - } diff --git a/iaai_scraper/browser/__init__.py b/iaai_scraper/browser/__init__.py deleted file mode 100644 index 38bb3f4..0000000 --- a/iaai_scraper/browser/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .factory import BrowserFactory -from .listing import ListingCollector -from .network import NetworkCapture -from .pace import HumanPacer - -__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"] diff --git a/iaai_scraper/browser/factory.py b/iaai_scraper/browser/factory.py deleted file mode 100644 index 69b393c..0000000 --- a/iaai_scraper/browser/factory.py +++ /dev/null @@ -1,212 +0,0 @@ -import json -import logging -import random - -from playwright.sync_api import Browser, BrowserContext, Playwright - -try: - from playwright_stealth import stealth_sync -except ImportError: - stealth_sync = None - -from ..core.config import Settings - -logger = logging.getLogger("iaai_scraper.browser") - - -def _build_init_script() -> str: - # Патч признаков автоматизации. - hardware_concurrency = random.choice([4, 8, 12, 16]) - device_memory = random.choice([4, 8, 16]) - languages = ["en-US", "en"] - - return f""" -(() => {{ - const define = (obj, prop, value) => {{ - try {{ - Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }}); - }} catch (e) {{}} - }}; - - define(navigator, 'webdriver', undefined); - define(navigator, 'platform', 'Win32'); - define(navigator, 'vendor', 'Google Inc.'); - define(navigator, 'language', '{languages[0]}'); - define(navigator, 'languages', {json.dumps(languages)}); - define(navigator, 'hardwareConcurrency', {hardware_concurrency}); - define(navigator, 'deviceMemory', {device_memory}); - define(navigator, 'maxTouchPoints', 0); - - if (!window.chrome) {{ - Object.defineProperty(window, 'chrome', {{ - value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }}, - configurable: true - }}); - }} - - const originalQuery = navigator.permissions && navigator.permissions.query; - if (originalQuery) {{ - navigator.permissions.query = (params) => ( - params && params.name === 'notifications' - ? Promise.resolve({{ state: Notification.permission }}) - : originalQuery(params) - ); - }} - - const originalGetParameter = WebGLRenderingContext.prototype.getParameter; - WebGLRenderingContext.prototype.getParameter = function(parameter) {{ - if (parameter === 37445) return 'Intel Inc.'; - if (parameter === 37446) return 'Intel Iris OpenGL Engine'; - return originalGetParameter.call(this, parameter); - }}; -}})(); -""" - - -class BrowserFactory: - - def __init__(self, settings: Settings) -> None: - self.settings = settings - - def _resolve_engine(self) -> str: - # Выбор движка браузера. - engine = self.settings.browser_engine.strip().lower() - if engine == "auto": - return "firefox" if self.settings.headless else "chromium" - if engine in ("firefox", "chromium"): - return engine - logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine) - return "firefox" if self.settings.headless else "chromium" - - def create_browser(self, playwright: Playwright) -> Browser: - engine = self._resolve_engine() - proxy_dict = self.settings.proxy.to_playwright_dict() - - if engine == "firefox": - launch_kwargs: dict = {"headless": self.settings.headless} - if proxy_dict: - launch_kwargs["proxy"] = proxy_dict - logger.info("Using proxy: %s", self.settings.proxy.server) - # Параметры Firefox для headless-режима. - launch_kwargs["firefox_user_prefs"] = { - "dom.webdriver.enabled": False, - "useAutomationExtension": False, - # Базовые оптимизации для VPS. - "media.autoplay.default": 5, - "media.volume_scale": "0.0", - "media.audio.playback.standalone": False, - "dom.ipc.processCount": 1, - "dom.ipc.plugins.enabled": False, - "browser.cache.disk.enable": False, - "browser.cache.memory.enable": True, - "browser.cache.memory.max_entry_size": 8192, - "network.prefetch-next": False, - "network.dns.disablePrefetch": True, - "permissions.default.image": 2, - "javascript.options.mem.gc_incremental_mark_slice_ms": 20, - } - logger.info("Launching Firefox (headless=%s)", self.settings.headless) - return playwright.firefox.launch(**launch_kwargs) - - # Путь запуска Chromium. - args = [ - "--disable-blink-features=AutomationControlled", - "--no-default-browser-check", - "--disable-dev-shm-usage", - "--disable-features=IsolateOrigins,site-per-process", - ] - if self.settings.headless: - args.append("--headless=new") - pw_headless = False - logger.info("Using Chromium new-headless mode (--headless=new)") - else: - pw_headless = False - - launch_kwargs = {"headless": pw_headless, "args": args} - if proxy_dict: - launch_kwargs["proxy"] = proxy_dict - logger.info("Using proxy: %s", self.settings.proxy.server) - try: - logger.info("Trying to launch real Chrome channel") - return playwright.chromium.launch(channel="chrome", **launch_kwargs) - except Exception: - logger.warning("Chrome channel launch failed, falling back to Chromium") - return playwright.chromium.launch(**launch_kwargs) - - def create_context(self, browser: Browser) -> BrowserContext: - viewport = random.choice(self.settings.fingerprint.viewport_presets) - timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) - color_scheme = random.choice(["light", "dark"]) - - is_firefox = browser.browser_type.name == "firefox" - - ctx_kwargs: dict = { - "viewport": viewport, - "screen": viewport, - "locale": self.settings.fingerprint.locale, - "timezone_id": timezone_id, - "color_scheme": color_scheme, - "java_script_enabled": True, - "ignore_https_errors": False, - } - - if is_firefox: - # Заголовки и user-agent для Firefox. - ctx_kwargs["user_agent"] = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " - "Gecko/20100101 Firefox/128.0" - ) - ctx_kwargs["extra_http_headers"] = { - "Accept-Language": "en-US,en;q=0.5", - "DNT": "1", - "Upgrade-Insecure-Requests": "1", - } - else: - ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent - ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25]) - ctx_kwargs["is_mobile"] = False - ctx_kwargs["has_touch"] = False - ctx_kwargs["extra_http_headers"] = { - "Accept-Language": "en-US,en;q=0.9", - "DNT": "1", - "Upgrade-Insecure-Requests": "1", - "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, - "Sec-CH-UA-Mobile": "?0", - "Sec-CH-UA-Platform": '"Windows"', - } - - context = browser.new_context(**ctx_kwargs) - context.set_default_timeout(self.settings.default_timeout_ms) - context.set_default_navigation_timeout(self.settings.default_timeout_ms) - - if not is_firefox: - # Патчи маскировки для Chromium. - context.add_init_script(_build_init_script()) - if stealth_sync: - context.on("page", lambda page: stealth_sync(page)) - logger.debug("playwright-stealth attached to context") - - return context - - @staticmethod - def enable_resource_blocking(page) -> None: - # Блокируем тяжёлые ресурсы для ускорения загрузки страниц. - BLOCKED_TYPES = {"image", "stylesheet", "font", "media"} - BLOCKED_URL_PATTERNS = ( - "google-analytics", "googletagmanager", "facebook.net", - "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2", - "analytics", "tracking", "adservice", - ) - - def _handle_route(route): - req = route.request - if req.resource_type in BLOCKED_TYPES: - route.abort() - return - url = req.url.lower() - if any(pat in url for pat in BLOCKED_URL_PATTERNS): - route.abort() - return - route.continue_() - - page.route("**/*", _handle_route) diff --git a/iaai_scraper/browser/listing.py b/iaai_scraper/browser/listing.py deleted file mode 100644 index 943bd84..0000000 --- a/iaai_scraper/browser/listing.py +++ /dev/null @@ -1,278 +0,0 @@ -import logging -import re -import time -from dataclasses import asdict, dataclass, field -from typing import Any -from urllib.parse import urljoin - -from playwright.sync_api import Page - -from .pace import HumanPacer -from ..core.config import Settings -from ..core.utils import first_non_empty - -logger = logging.getLogger("iaai_scraper.listing") -VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) - - -@dataclass(slots=True) -class ListingVehicleLink: - href: str - title: str = "" - lot_number: str | None = None - - -@dataclass(slots=True) -class ListingPageResult: - source_url: str - page_number: int - vehicle_links: list[ListingVehicleLink] = field(default_factory=list) - pagination_available: bool = False - next_page_detected: bool = False - - -class ListingCollector: - def __init__(self, settings: Settings, pacer: HumanPacer) -> None: - self.settings = settings - self.pacer = pacer - - def open_cars_listing(self, page: Page) -> None: - logger.info("Opening cars listing page: %s", self.settings.listing.cars_url) - url = self.settings.listing.cars_url - last_err = None - for attempt in range(3): - try: - page.goto(url, wait_until="commit", timeout=60_000) - last_err = None - break - except Exception as e: - last_err = e - logger.warning("goto listing attempt %d failed: %s", attempt + 1, e) - # Небольшой backoff при ошибках открытия листинга. - time.sleep(5 * (attempt + 1)) - if last_err: - logger.warning("All goto attempts failed, trying JS navigation") - try: - page.evaluate(f"window.location.href = '{url}'") - except Exception: - pass - # Быстрая проверка готовности страницы. - try: - page.wait_for_load_state("domcontentloaded", timeout=12_000) - except Exception: - pass - try: - page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=4_000) - except Exception: - # Короткая пауза вместо длинного sleep. - time.sleep(0.25) - logger.info("Listing page URL: %s", page.url) - self.pacer.after_listing_open() - - def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]: - applied = {"make": None, "model": None} - if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): - applied["make"] = make - self.pacer.after_filter_action() - if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model): - applied["model"] = model - self.pacer.after_filter_action() - return applied - - def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: - # Считываем ссылки одним проходом по DOM. - raw_items = page.eval_on_selector_all( - "a[href*='/VehicleDetail/']", - """ - (nodes) => nodes.map((a) => ({ - href: a.getAttribute('href') || '', - title: a.getAttribute('title') || '', - text: (a.textContent || '').trim(), - })) - """, - ) - total = min(len(raw_items), self.settings.listing.page_link_limit) - links: list[ListingVehicleLink] = [] - seen: set[str] = set() - for idx in range(total): - item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} - href = str(item.get("href") or "") - match = VEHICLE_HREF_RE.search(href) - if not match: - continue - lot_number = match.group(1) - absolute = urljoin(self.settings.home_url, match.group(0)) - if absolute in seen: - continue - seen.add(absolute) - title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" - links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) - if len(links) >= self.settings.listing.max_vehicles_per_run: - break - next_page_detected = self._has_next_page(page) - return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) - - def go_to_next_page(self, page: Page) -> bool: - # Запоминаем первую ссылку текущей страницы для определения смены контента. - old_first_href = "" - try: - first_link = page.locator("a[href*='/VehicleDetail/']").first - if first_link.count() > 0: - old_first_href = first_link.get_attribute("href") or "" - except Exception: - pass - - selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"] - for selector in selectors: - locator = page.locator(selector).first - if locator.count() == 0: - continue - disabled = (locator.get_attribute("disabled") or "").lower() - aria_disabled = (locator.get_attribute("aria-disabled") or "").lower() - classes = (locator.get_attribute("class") or "").lower() - if disabled or aria_disabled == "true" or "disabled" in classes: - continue - self.pacer.move_mouse_to(page, locator) - locator.click() - - # Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться. - if old_first_href: - try: - page.wait_for_function( - f"""() => {{ - const a = document.querySelector("a[href*='/VehicleDetail/']"); - return a && a.getAttribute('href') !== '{old_first_href}'; - }}""", - timeout=8000, - ) - except Exception: - pass - else: - try: - page.wait_for_load_state("domcontentloaded", timeout=15000) - except Exception: - pass - - try: - page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000) - except Exception: - pass - self.pacer.after_page_change() - return True - return False - - def collect_listing_links( - self, - page: Page, - *, - make: str | None = None, - model: str | None = None, - known_origin_ids: set[str] | None = None, - ) -> dict[str, Any]: - self.open_cars_listing(page) - applied_filters = self.apply_filters(page, make=make, model=model) - pages: list[dict[str, object]] = [] - all_links: list[str] = [] - early_stopped = False - threshold = self.settings.listing.early_stop_threshold - - for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): - page_result = self.collect_current_page(page, page_number=page_number) - pages.append({ - "page_number": page_result.page_number, - "source_url": page_result.source_url, - "links_found": len(page_result.vehicle_links), - "vehicle_links": [asdict(item) for item in page_result.vehicle_links], - "next_page_detected": page_result.next_page_detected, - }) - for item in page_result.vehicle_links: - if item.href not in all_links: - all_links.append(item.href) - if len(all_links) >= self.settings.listing.max_vehicles_per_run: - break - - # Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла. - # Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице). - if ( - known_origin_ids is not None - and threshold > 0.0 - and page_result.vehicle_links - ): - page_known = sum( - 1 for item in page_result.vehicle_links - if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids - ) - page_new = len(page_result.vehicle_links) - page_known - ratio = page_known / len(page_result.vehicle_links) - # Останавливаемся только если нет новых И порог превышен - if ratio >= threshold and page_new == 0: - logger.info( - "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", - page_number, ratio * 100, page_known, - len(page_result.vehicle_links), threshold * 100, - ) - early_stopped = True - break - elif page_new > 0 and ratio >= threshold: - logger.info( - "Page %d: %.0f%% known but %d new found — продолжаем", - page_number, ratio * 100, page_new, - ) - - if ( - len(all_links) >= self.settings.listing.max_vehicles_per_run - or self.settings.listing.collect_current_page_only - or not self.settings.listing.include_pagination - or not page_result.next_page_detected - ): - break - if not self.go_to_next_page(page): - break - - return { - "listing_url": self.settings.listing.cars_url, - "applied_filters": applied_filters, - "pages_collected": len(pages), - "vehicles_collected": len(all_links), - "vehicle_urls": all_links, - "early_stopped": early_stopped, - "pages": pages, - "strategy": { - "sequential": True, - "collect_current_page_only": self.settings.listing.collect_current_page_only, - "include_pagination": self.settings.listing.include_pagination, - "max_pages_per_run": self.settings.listing.max_pages_per_run, - "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run, - "early_stop_threshold": threshold, - }, - } - - @staticmethod - def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: - for selector in selectors: - locator = page.locator(selector).first - if locator.count() == 0: - continue - try: - locator.click() - locator.fill(value) - page.keyboard.press("Enter") - try: - page.wait_for_load_state("domcontentloaded", timeout=15000) - except Exception: - pass - try: - page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000) - except Exception: - pass - return True - except Exception: - continue - return False - - @staticmethod - def _has_next_page(page: Page) -> bool: - for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]: - if page.locator(selector).count() > 0: - return True - return False diff --git a/iaai_scraper/browser/network.py b/iaai_scraper/browser/network.py deleted file mode 100644 index 0674976..0000000 --- a/iaai_scraper/browser/network.py +++ /dev/null @@ -1,130 +0,0 @@ -import json -import logging -from dataclasses import dataclass, field -from typing import Any -from urllib.parse import urlparse - -from playwright.sync_api import Page, Request, Response - -from ..core.config import Settings - -logger = logging.getLogger("iaai_scraper.network") - - -@dataclass -class NetworkCapture: - # Перехватчик сетевых запросов. - - settings: Settings - requests: list[dict[str, Any]] = field(default_factory=list) - json_responses: list[dict[str, Any]] = field(default_factory=list) - _seen_req: set[str] = field(default_factory=set) - _seen_resp: set[str] = field(default_factory=set) - _origin: str | None = None - _page: Any = field(default=None) - - def attach(self, page: Page, origin_url: str | None = None) -> None: - # Снимаем старые подписки перед повторным attach. - try: - page.remove_listener("request", self._on_request) - page.remove_listener("response", self._on_response) - except Exception: - pass - # Подписка на сетевые события - try: - self._origin = urlparse(origin_url or page.url).netloc.lower() or None - except Exception: - self._origin = None - self._page = page - page.on("request", self._on_request) - page.on("response", self._on_response) - - def _is_same_origin(self, url: str) -> bool: - # Фильтр по домену - if not self.settings.capture.capture_same_origin_only or not self._origin: - return True - netloc = urlparse(url).netloc.lower() - return netloc == self._origin or netloc.endswith(".iaai.com") - - def _on_request(self, request: Request) -> None: - # Берём только xhr/fetch - if request.resource_type not in {"xhr", "fetch"}: - return - if not self._is_same_origin(request.url): - return - if len(self.requests) >= self.settings.capture.max_requests: - return - key = f"{request.method}:{request.url}:{request.post_data or ''}" - # Убираем дубли - if key in self._seen_req: - return - self._seen_req.add(key) - self.requests.append({ - "url": request.url, "method": request.method, - "resource_type": request.resource_type, "post_data": request.post_data, - }) - - def _on_response(self, response: Response) -> None: - # Сохраняем JSON - request = response.request - if request.resource_type not in {"xhr", "fetch"}: - return - if not self._is_same_origin(response.url): - return - if len(self.json_responses) >= self.settings.capture.max_json_responses: - return - if not self._is_json(response): - return - key = f"{request.method}:{response.url}:{response.status}" - if key in self._seen_resp: - return - self._seen_resp.add(key) - try: - payload = response.json() - except Exception: - try: - payload = json.loads(response.text()) - except Exception: - return - self.json_responses.append({ - "url": response.url, "status": response.status, - "request_method": request.method, "post_data": request.post_data, - "resource_type": request.resource_type, "payload": payload, - "category": self._categorize(response.url), - }) - - @staticmethod - def _is_json(response: Response) -> bool: - ct = (response.headers.get("content-type") or "").lower() - if "application/json" in ct or "+json" in ct: - return True - url = response.url.lower() - return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"]) - - @staticmethod - def _categorize(url: str) -> str: - # Простая категория URL - low = url.lower() - mapping = { - "images": ["image", "media", "photos", "gallery"], - "bids": ["bid", "offer", "buy-now", "buynow"], - "auction": ["auction", "sale", "lane", "branch"], - "vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"], - "documents": ["title", "document", "report"], - } - for cat, markers in mapping.items(): - if any(m in low for m in markers): - return cat - return "other" - - def export(self) -> dict[str, Any]: - responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"])) - return { - "requests": self.requests, - "json_responses": responses, - "capture_limits": { - "same_origin_only": self.settings.capture.capture_same_origin_only, - "max_requests": self.settings.capture.max_requests, - "max_json_responses": self.settings.capture.max_json_responses, - }, - } diff --git a/iaai_scraper/browser/pace.py b/iaai_scraper/browser/pace.py deleted file mode 100644 index 10c2551..0000000 --- a/iaai_scraper/browser/pace.py +++ /dev/null @@ -1,46 +0,0 @@ -import random -import time - -from playwright.sync_api import Locator, Page - -from ..core.config import Settings - - -class HumanPacer: - # Случайные паузы между действиями. - - def __init__(self, settings: Settings) -> None: - self.settings = settings - - def pause(self, min_s: float, max_s: float) -> None: - if self.settings.pace.enabled: - time.sleep(random.uniform(min_s, max_s)) - - def after_listing_open(self) -> None: - self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s) - - def after_filter_action(self) -> None: - self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s) - - def before_vehicle_open(self) -> None: - self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s) - - def after_vehicle_open(self) -> None: - self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s) - - def between_vehicles(self) -> None: - self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s) - - def after_page_change(self) -> None: - self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s) - - def move_mouse_to(self, page: Page, locator: Locator) -> None: - try: - box = locator.bounding_box() - except Exception: - box = None - if not box: - return - x = box["x"] + box["width"] * random.uniform(0.2, 0.8) - y = box["y"] + box["height"] * random.uniform(0.2, 0.8) - page.mouse.move(x, y, steps=random.randint(8, 18)) diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py deleted file mode 100644 index c3b76d5..0000000 --- a/iaai_scraper/cli.py +++ /dev/null @@ -1,82 +0,0 @@ -import argparse -from pathlib import Path - -from .core.config import Settings -from .core.utils import save_to_json -from .scraper import IAAIScraper - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="IAAI scraper CLI") - parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") - parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") - subparsers = parser.add_subparsers(dest="command", required=True) - - default_output_dir = Path("artifacts/json") - - subparsers.add_parser("init-db", help="Create DB tables") - - listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page") - listing_parser.add_argument("--make", default=None) - listing_parser.add_argument("--model", default=None) - listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json")) - - scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page") - scrape_parser.add_argument("vehicle_url") - scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json")) - - sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle") - sync_vehicle_parser.add_argument("vehicle_url") - sync_vehicle_parser.add_argument("--lane", default="iaai") - sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json")) - - sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles") - sync_listing_parser.add_argument("--make", default=None) - sync_listing_parser.add_argument("--model", default=None) - sync_listing_parser.add_argument("--lane", default="iaai_cars") - sync_listing_parser.add_argument("--limit", type=int, default=None) - sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) - sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json")) - - return parser - - -def main() -> None: - parser = build_parser() - args = parser.parse_args() - - runtime_settings: Settings | None = None - if args.headless is not None or args.debug: - runtime_settings = Settings() - if args.headless is not None: - runtime_settings.headless = args.headless == "true" - if args.debug: - runtime_settings.log_level = "DEBUG" - - with IAAIScraper(runtime_settings) as scraper: - if args.command == "init-db": - data = scraper.init_db() - print(f"DB initialized: {data}") - return - elif args.command == "collect-listing": - data = scraper.collect_listing(make=args.make, model=args.model) - elif args.command == "scrape-vehicle": - data = scraper.scrape_vehicle_detail(args.vehicle_url) - elif args.command == "sync-vehicle": - data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) - else: - only_new = None if args.only_new is None else args.only_new == "true" - data = scraper.sync_listing( - make=args.make, - model=args.model, - lane=args.lane, - limit=args.limit, - only_new=only_new, - ) - - save_to_json(data, Path(args.output)) - print(f"Saved to {Path(args.output).resolve()}") - - -if __name__ == "__main__": - main() diff --git a/iaai_scraper/core/__init__.py b/iaai_scraper/core/__init__.py deleted file mode 100644 index c9c2ef6..0000000 --- a/iaai_scraper/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__: list[str] = [] diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py deleted file mode 100644 index 4690527..0000000 --- a/iaai_scraper/core/config.py +++ /dev/null @@ -1,218 +0,0 @@ -import os -from dataclasses import dataclass, field -from pathlib import Path - -from dotenv import load_dotenv - -load_dotenv() - - -TRUE_VALUES = {"1", "true", "yes", "on"} - - -# Хелперы для чтения env-переменных с приведением типов - -def _env_str(name: str, default: str) -> str: - value = os.getenv(name) - return value if value is not None else default - - -def _env_optional_str(name: str) -> str | None: - value = os.getenv(name) - if value is None: - return None - value = value.strip() - return value or None - - -def _env_path_str(name: str) -> str | None: - value = _env_optional_str(name) - if value is None: - return None - return str(Path(value).expanduser()) - - -def _env_bool(name: str, default: bool) -> bool: - fallback = "true" if default else "false" - return _env_str(name, fallback).strip().lower() in TRUE_VALUES - - -def _env_int(name: str, default: int) -> int: - return int(_env_str(name, str(default)).strip()) - - -def _env_float(name: str, default: float) -> float: - return float(_env_str(name, str(default)).strip()) - - -# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) - -@dataclass(slots=True) -class FingerprintConfig: - user_agent: str = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/135.0.0.0 Safari/537.36" - ) - viewport_presets: tuple[dict[str, int], ...] = ( - {"width": 1920, "height": 1080}, - {"width": 1600, "height": 900}, - {"width": 1536, "height": 864}, - {"width": 1440, "height": 900}, - {"width": 1366, "height": 768}, - ) - timezone_candidates: tuple[str, ...] = ( - "America/New_York", - "America/Chicago", - "America/Los_Angeles", - ) - locale: str = "en-US" - sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' - - -# Конфиг перехвата сетевых запросов (лимиты на кол-во) - -@dataclass(slots=True) -class CaptureConfig: - capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True) - max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40) - max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30) - - -# Конфиг пауз между действиями (имитация человека) - -@dataclass(slots=True) -class HumanPaceConfig: - enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True) - after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5) - after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2) - after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5) - after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2) - before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) - before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) - after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05) - after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15) - between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05) - between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15) - after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8) - after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8) - - -# Конфиг сбора листинга (URL, лимиты страниц и машин) - -@dataclass(slots=True) -class ListingConfig: - cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") - max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999) - max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000) - page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500) - include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True) - collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False) - # Порог раннего останова: если доля уже известных машин на странице >= этого значения, - # прекращаем листать — все новые машины уже найдены. 0 = отключено. - early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8) - - -# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) - -@dataclass(slots=True) -class DatabaseConfig: - url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper") - echo: bool = _env_bool("IAAI_DATABASE_ECHO", False) - pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5) - max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10) - pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800) - auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False) - - -# --- Конфиг Redis (URL для Celery broker) --- - -@dataclass(slots=True) -class RedisConfig: - url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0") - - -# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- - -@dataclass(slots=True) -class CeleryConfig: - broker_url: str = _env_str("CELERY_BROKER_URL", "") - result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") - task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) - task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) - worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) - worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) - broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) - beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) - beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None - batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) - parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8) - block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True) - - -# --- Конфиг прокси (server, username, password) --- - -@dataclass(slots=True) -class ProxyConfig: - server: str | None = _env_optional_str("IAAI_PROXY_SERVER") - username: str | None = _env_optional_str("IAAI_PROXY_USERNAME") - password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD") - - @property - def enabled(self) -> bool: - return bool(self.server) - - def to_playwright_dict(self) -> dict[str, str] | None: - if not self.server: - return None - result: dict[str, str] = {"server": self.server} - if self.username: - result["username"] = self.username - if self.password: - result["password"] = self.password - return result - - -# --- Главный объект настроек: собирает все блоки конфигурации --- - -@dataclass(slots=True) -class Settings: - home_url: str = "https://www.iaai.com/" - default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000) - network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400) - fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000) - fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1) - fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000) - max_retries: int = _env_int("IAAI_MAX_RETRIES", 3) - retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5) - retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0) - retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25) - headless: bool = _env_bool("IAAI_HEADLESS", True) - browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto") - log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO") - log_file: str | None = _env_optional_str("IAAI_LOG_FILE") - enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True) - sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", True) - raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON") - tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE") - runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE") - scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60) - fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) - capture: CaptureConfig = field(default_factory=CaptureConfig) - pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) - listing: ListingConfig = field(default_factory=ListingConfig) - database: DatabaseConfig = field(default_factory=DatabaseConfig) - redis: RedisConfig = field(default_factory=RedisConfig) - celery: CeleryConfig = field(default_factory=CeleryConfig) - proxy: ProxyConfig = field(default_factory=ProxyConfig) - - @property - def parallel_tabs(self) -> int: - return self.celery.parallel_tabs - - @property - def block_resources(self) -> bool: - return self.celery.block_resources - -# Глобальный синглтон — используется по умолчанию во всех модулях. -settings = Settings() diff --git a/iaai_scraper/core/exceptions.py b/iaai_scraper/core/exceptions.py deleted file mode 100644 index f984981..0000000 --- a/iaai_scraper/core/exceptions.py +++ /dev/null @@ -1,10 +0,0 @@ -class ScraperError(Exception): - """Базовое исключение скрапера.""" - - -class AntiBotDetectedError(ScraperError): - """Вызывается, когда сайт блокирует автоматизацию.""" - - -class SiteStructureChangedError(ScraperError): - """Вызывается, когда структура страницы изменилась и данных не хватает.""" diff --git a/iaai_scraper/core/logs.py b/iaai_scraper/core/logs.py deleted file mode 100644 index 32a312d..0000000 --- a/iaai_scraper/core/logs.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging -import sys -from contextvars import ContextVar - -# Храним trace_id текущего потока/корутины. -TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") - - -class TraceIdFilter(logging.Filter): - # Добавляет trace_id в каждую запись лога для сквозной трассировки. - def filter(self, record: logging.LogRecord) -> bool: - record.trace_id = TRACE_ID.get() - return True - - -def set_trace_id(trace_id: str) -> None: - TRACE_ID.set(trace_id) - - -def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: - handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)] - if log_file: - handlers.append(logging.FileHandler(log_file, encoding="utf-8")) - trace_filter = TraceIdFilter() - for handler in handlers: - handler.addFilter(trace_filter) - logging.basicConfig( - level=getattr(logging, level.upper(), logging.INFO), - format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s", - handlers=handlers, - force=True, - ) diff --git a/iaai_scraper/core/retry.py b/iaai_scraper/core/retry.py deleted file mode 100644 index 2fce32c..0000000 --- a/iaai_scraper/core/retry.py +++ /dev/null @@ -1,53 +0,0 @@ -import logging -import random -import time -from collections.abc import Callable -from functools import wraps -from typing import Any - -from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError - -from .exceptions import AntiBotDetectedError - -logger = logging.getLogger("iaai_scraper.retry") - -# Типы исключений, при которых retry имеет смысл. -RETRYABLE_EXCEPTIONS = ( - PlaywrightTimeoutError, - Error, - ConnectionError, - OSError, - TimeoutError, - AntiBotDetectedError, -) - - -def retryable( - max_attempts: int, - delay_seconds: float = 2.5, - backoff_multiplier: float = 2.0, - jitter_seconds: float = 0.0, -) -> Callable[[Callable[..., Any]], Callable[..., Any]]: - def decorator(func: Callable[..., Any]) -> Callable[..., Any]: - @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: - last_error: Exception | None = None - for attempt in range(1, max_attempts + 1): - try: - return func(*args, **kwargs) - except RETRYABLE_EXCEPTIONS as exc: - last_error = exc - logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc) - if attempt < max_attempts: - sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1)) - if jitter_seconds > 0: - sleep_for += random.uniform(0, jitter_seconds) - logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for) - time.sleep(sleep_for) - if last_error is not None: - raise last_error - raise RuntimeError("Retry wrapper failed without a captured exception") - - return wrapper - - return decorator diff --git a/iaai_scraper/core/runtime_config.py b/iaai_scraper/core/runtime_config.py deleted file mode 100644 index 205a154..0000000 --- a/iaai_scraper/core/runtime_config.py +++ /dev/null @@ -1,301 +0,0 @@ -import json -import logging -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - - -logger = logging.getLogger("iaai_scraper.runtime_config") - - -def _normalize_text(value: str) -> str: - return value.strip().casefold() - - -def _text_tuple(values: Any) -> tuple[str, ...]: - return tuple( - str(item).strip() - for item in (values or []) - if str(item).strip() - ) - - -def _int_tuple(values: Any) -> tuple[int, ...]: - result: list[int] = [] - for value in values or []: - try: - result.append(int(value)) - except (TypeError, ValueError): - continue - return tuple(result) - - -def _optional_int(value: Any) -> int | None: - if value in (None, ""): - return None - try: - return int(value) - except (TypeError, ValueError): - return None - - -def _optional_bool(value: Any) -> bool | None: - if value is None: - return None - if isinstance(value, bool): - return value - if isinstance(value, str): - normalized = value.strip().casefold() - if normalized in {"1", "true", "yes", "on"}: - return True - if normalized in {"0", "false", "no", "off"}: - return False - return None - - -@dataclass(slots=True) -class RuntimeSyncConfig: - name: str | None = None - ids_initial_size: int | None = None - ids_next_size: int | None = None - ids_max_pages: int | None = None - condition_check_enabled: bool | None = None - lane: str | None = None - only_new: bool | None = None - limit: int | None = None - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig": - data = data or {} - name = str(data.get("name")).strip() if data.get("name") else None - lane = str(data.get("lane")).strip() if data.get("lane") else None - return cls( - name=name or None, - ids_initial_size=_optional_int(data.get("ids_initial_size")), - ids_next_size=_optional_int(data.get("ids_next_size")), - ids_max_pages=_optional_int(data.get("ids_max_pages")), - condition_check_enabled=_optional_bool(data.get("condition_check_enabled")), - lane=lane or None, - only_new=_optional_bool(data.get("only_new")), - limit=_optional_int(data.get("limit")), - ) - - -@dataclass(slots=True) -class RuntimeListingConfig: - make: str | None = None - model: str | None = None - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig": - data = data or {} - make = str(data.get("make")).strip() if data.get("make") else None - model = str(data.get("model")).strip() if data.get("model") else None - return cls(make=make or None, model=model or None) - - -@dataclass(slots=True) -class RuntimeFieldFilters: - brands: tuple[str, ...] = () - models: tuple[str, ...] = () - years: tuple[int, ...] = () - body_types: tuple[str, ...] = () - colors: tuple[str, ...] = () - drives: tuple[str, ...] = () - gearboxes: tuple[str, ...] = () - locations: tuple[str, ...] = () - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters": - data = data or {} - return cls( - brands=_text_tuple(data.get("brands")), - models=_text_tuple(data.get("models")), - years=_int_tuple(data.get("years")), - body_types=_text_tuple(data.get("body_types")), - colors=_text_tuple(data.get("colors")), - drives=_text_tuple(data.get("drives")), - gearboxes=_text_tuple(data.get("gearboxes")), - locations=_text_tuple(data.get("locations")), - ) - - def is_empty(self) -> bool: - return not any([ - self.brands, - self.models, - self.years, - self.body_types, - self.colors, - self.drives, - self.gearboxes, - self.locations, - ]) - - def matches(self, values: dict[str, Any]) -> bool: - return all([ - self._match_text(self.brands, values.get("brand")), - self._match_text(self.models, values.get("model")), - self._match_int(self.years, values.get("year")), - self._match_text(self.body_types, values.get("body_type")), - self._match_text(self.colors, values.get("color")), - self._match_text(self.drives, values.get("drive")), - self._match_text(self.gearboxes, values.get("gearbox")), - self._match_text(self.locations, values.get("location")), - ]) - - @staticmethod - def _match_text(allowed: tuple[str, ...], value: Any) -> bool: - if not allowed: - return True - normalized = _normalize_text(str(value or "")) - return normalized in {_normalize_text(item) for item in allowed} - - @staticmethod - def _match_int(allowed: tuple[int, ...], value: Any) -> bool: - if not allowed: - return True - parsed = _optional_int(value) - return parsed in set(allowed) - - -@dataclass(slots=True) -class RuntimeRangeFilter: - min: int | None = None - max: int | None = None - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter": - data = data or {} - return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max"))) - - def is_empty(self) -> bool: - return self.min is None and self.max is None - - def matches(self, value: Any) -> bool: - parsed = _optional_int(value) - if parsed is None: - return self.is_empty() - if self.min is not None and parsed < self.min: - return False - if self.max is not None and parsed > self.max: - return False - return True - - -@dataclass(slots=True) -class RuntimeFlagFilters: - damaged_only: bool | None = None - run_and_drive: bool | None = None - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters": - data = data or {} - return cls( - damaged_only=_optional_bool(data.get("damaged_only")), - run_and_drive=_optional_bool(data.get("run_and_drive")), - ) - - def is_empty(self) -> bool: - return self.damaged_only is None and self.run_and_drive is None - - def matches(self, values: dict[str, Any]) -> bool: - if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only: - return False - if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive: - return False - return True - - -@dataclass(slots=True) -class RuntimeFiltersConfig: - include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) - exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters) - price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) - mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter) - flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters) - - @classmethod - def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig": - data = data or {} - legacy_fields = RuntimeFieldFilters.from_dict(data) - include_payload = data.get("include") - exclude_payload = data.get("exclude") - - flat_exclude_payload = { - "brands": data.get("exclude_brands"), - "models": data.get("exclude_models"), - "years": data.get("exclude_years"), - "body_types": data.get("exclude_body_types"), - "colors": data.get("exclude_colors"), - "drives": data.get("exclude_drives"), - "gearboxes": data.get("exclude_gearboxes"), - "locations": data.get("exclude_locations"), - } - - include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields - exclude = ( - RuntimeFieldFilters.from_dict(exclude_payload) - if exclude_payload is not None - else RuntimeFieldFilters.from_dict(flat_exclude_payload) - ) - - return cls( - include=include, - exclude=exclude, - price=RuntimeRangeFilter.from_dict(data.get("price")), - mileage=RuntimeRangeFilter.from_dict(data.get("mileage")), - flags=RuntimeFlagFilters.from_dict(data.get("flags")), - ) - - def is_empty(self) -> bool: - return all([ - self.include.is_empty(), - self.exclude.is_empty(), - self.price.is_empty(), - self.mileage.is_empty(), - self.flags.is_empty(), - ]) - - def matches(self, values: dict[str, Any]) -> bool: - if not self.include.matches(values): - return False - if not self._matches_exclude(values): - return False - if not self.price.matches(values.get("price")): - return False - if not self.mileage.matches(values.get("mileage")): - return False - if not self.flags.matches(values): - return False - return True - - def _matches_exclude(self, values: dict[str, Any]) -> bool: - if self.exclude.is_empty(): - return True - return not self.exclude.matches(values) - - -@dataclass(slots=True) -class RuntimeConfig: - sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig) - listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig) - filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig) - - @classmethod - def from_file(cls, config_path: str | None) -> "RuntimeConfig": - if not config_path: - return cls() - path = Path(config_path) - if not path.exists(): - logger.info("Runtime config file not found: %s", path) - return cls() - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except Exception as exc: - logger.warning("Failed to read runtime config %s: %s", path, exc) - return cls() - return cls( - sync=RuntimeSyncConfig.from_dict(payload.get("sync")), - listing=RuntimeListingConfig.from_dict(payload.get("listing")), - filters=RuntimeFiltersConfig.from_dict(payload.get("filters")), - ) diff --git a/iaai_scraper/core/utils.py b/iaai_scraper/core/utils.py deleted file mode 100644 index 8c8cad5..0000000 --- a/iaai_scraper/core/utils.py +++ /dev/null @@ -1,72 +0,0 @@ -import json -import re -from pathlib import Path -from typing import Any, Callable, Iterable - - -def save_to_json(data: Any, filename: str | Path) -> None: - path = Path(filename) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") - - -def first_non_empty(values: Iterable[Any]) -> Any | None: - for value in values: - if value not in (None, "", [], {}, ()): - return value - return None - - -# Регулярные выражения для VIN, lot и price. -VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) -LOT_RE = re.compile(r"\b(\d{7,10})\b") -PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") - - -def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: - # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве. - found = [] - if _depth >= max_depth: - return found - if isinstance(obj, dict): - for key, value in obj.items(): - if key.lower() in target_keys: - found.append(value) - found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) - elif isinstance(obj, list): - for item in obj: - found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) - return found - - -def deep_find_all_keys( - payloads: list, - field_map: dict[str, set[str]], - max_depth: int = 64, -) -> dict[str, list]: - """Извлекает все нужные поля за один проход по JSON.""" - # Готовим обратную карту: нормализованный ключ -> имя поля. - reverse: dict[str, str] = {} - for field_name, keys in field_map.items(): - for k in keys: - reverse[k.lower()] = field_name - - result: dict[str, list] = {f: [] for f in field_map} - - def _recurse(obj: Any, depth: int) -> None: - if depth >= max_depth: - return - if isinstance(obj, dict): - for k, v in obj.items(): - field = reverse.get(k.lower()) - if field is not None: - result[field].append(v) - _recurse(v, depth + 1) - elif isinstance(obj, list): - for item in obj: - _recurse(item, depth + 1) - - for payload in payloads: - _recurse(payload, 0) - - return result diff --git a/iaai_scraper/parsing/__init__.py b/iaai_scraper/parsing/__init__.py deleted file mode 100644 index c9c2ef6..0000000 --- a/iaai_scraper/parsing/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__: list[str] = [] diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py deleted file mode 100644 index 2c1f79f..0000000 --- a/iaai_scraper/parsing/mapper.py +++ /dev/null @@ -1,405 +0,0 @@ -import hashlib -import re -from datetime import datetime, timezone -from string import ascii_letters, digits -from typing import Any -from urllib.parse import urlparse - -from ..core.utils import first_non_empty -from ..storage.enums import ( - BODY_TYPE_ENUM_VALUES, - COUNTRY_ENUM_VALUES, - CURRENCY_ENUM_VALUES, - DRIVE_ENUM_VALUES, - GEARBOX_ENUM_VALUES, - ORIGIN_ENUM_VALUES, - SELLING_TYPE_ENUM_VALUES, - STEERING_WHEEL_ENUM_VALUES, -) -from ..storage.schemas import CarRecord, ImageRecord - - -class CarMapper: - # Преобразование данных IAAI в CarRecord. - - BODY_MAP = { - "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", - "suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP", - "pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP", - "quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP", - "supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN", - "minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV", - } - DRIVE_MAP = { - "front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD", - "all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD", - "4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD", - } - GEARBOX_MAP = { - "automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT", - "manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT", - "continuously variable transmission": "CVT", "electric": "EV", "ev": "EV", - } - STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"} - COUNTRY_MAP = { - "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", - "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", - } - NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} - - def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: - # Собираем нормализованную DB-модель. - vehicle_summary = vehicle_summary or {} - payload_insights = payload_insights or {} - core = payload_insights.get("vehicle_core", {}) - pricing = payload_insights.get("pricing", {}) - damage = payload_insights.get("damage", {}) - auction = payload_insights.get("auction", {}) - images = payload_insights.get("images", {}) - - origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) - parser_id = self._generate_parser_id(origin_id) - brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" - model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" - year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")])) - price = self._first_parsed_int( - [ - pricing.get("buy_now"), - pricing.get("current_bid"), - vehicle_summary.get("buy_now"), - vehicle_summary.get("current_bid"), - pricing.get("actual_cash_value"), - ], - self._to_money_int, - ) - mileage = self._parse_odometer( - first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")]) - ) - color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) - drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) - # Пытаемся определить привод из строки двигателя. - if not drive or drive == "NA": - engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")])) - if engine_text: - inferred_drive = self._normalize_drive(engine_text) - if inferred_drive and inferred_drive != "NA": - drive = inferred_drive - gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) - steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" - body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) - engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")])) - title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) - seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) - location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""])) - country = self._normalize_country(first_non_empty([core.get("country"), location, "US"])) - is_damaged = self._bool_damage(damage, vehicle_summary) - is_sold = self._bool_sold(auction) - one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) - new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False])) - rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False])) - repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False])) - non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True])) - evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None - currency = self._normalize_currency( - first_non_empty( - [ - pricing.get("currency"), - vehicle_summary.get("currency"), - pricing.get("buy_now"), - pricing.get("current_bid"), - vehicle_summary.get("buy_now"), - vehicle_summary.get("current_bid"), - "USD", - ] - ) - ) - slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))) - images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) - origin = "IAAI" - - return CarRecord( - parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, - mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, - steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, - selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car, - is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, - evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, - slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records, - ) - - @staticmethod - def _as_str(value: Any) -> str: - return "" if value is None else str(value).strip() - - @staticmethod - def _to_int(value: Any) -> int | None: - if value is None: - return None - if isinstance(value, bool): - return int(value) - if isinstance(value, (int, float)): - return int(value) - digits = re.sub(r"[^\d]", "", str(value)) - return int(digits) if digits else None - - @classmethod - def _to_money_int(cls, value: Any) -> int | None: - # Нормализация стоимости из разных форматов. - if value is None: - return None - if isinstance(value, bool): - return None - if isinstance(value, (int, float)): - return int(value) - - text = str(value).strip() - if not text: - return None - - lowered = text.lower() - if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]): - return None - - numbers = re.findall(r"\d[\d\s.,]*", text) - if not numbers: - return None - - best: int | None = None - for number in numbers: - clean = number.replace(" ", "") - if "," in clean and "." in clean: - # Поддержка 1,234.56 и 1.234,56. - if clean.rfind(",") > clean.rfind("."): - clean = clean.replace(".", "").replace(",", ".") - else: - clean = clean.replace(",", "") - elif "," in clean: - parts = clean.split(",") - # 123,45 -> 123.45, иначе разделитель тысяч. - if len(parts[-1]) in {1, 2} and len(parts) == 2: - clean = clean.replace(",", ".") - else: - clean = clean.replace(",", "") - elif "." in clean: - parts = clean.split(".") - if not (len(parts[-1]) in {1, 2} and len(parts) == 2): - clean = clean.replace(".", "") - - try: - parsed = int(float(clean)) - except ValueError: - continue - - if parsed > 0 and (best is None or parsed > best): - best = parsed - - return best - - @staticmethod - def _first_parsed_int(values: list[Any], parser) -> int | None: - for value in values: - parsed = parser(value) - if parsed is not None: - return parsed - return None - - @staticmethod - def _to_year(value: Any) -> int | None: - parsed = CarMapper._to_int(value) - if parsed is None: - return None - if 1900 <= parsed <= 2100: - return parsed - return None - - @staticmethod - def _parse_odometer(value: Any) -> int: - # Разбор пробега из строк IAAI. - if value is None: - return 0 - text = str(value).strip() - if not text: - return 0 - lowered = text.lower() - if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]): - return 0 - # Извлекаем число из строкового формата одометра. - numbers = re.findall(r"[\d,]+", text) - for num_str in numbers: - clean = num_str.replace(",", "") - if clean.isdigit() and int(clean) > 0: - return int(clean) - return 0 - - def _to_engine_cc(self, value: Any) -> int | None: - # Поддержка литров и cc. - text = str(value).lower().strip() if value is not None else "" - if not text: - return None - if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): - return int(float(liters_match.group(1)) * 1000) - if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text): - return int(cubic_match.group(1)) - return int(text) if re.match(r"^\d+$", text) else None - - def _normalize_currency(self, value: Any) -> str: - text = self._as_str(value) - upper = text.upper() or "USD" - if any(token in text for token in ["€", "EUR"]): - return "EUR" - if any(token in text for token in ["¥", "JPY"]): - return "JPY" - if any(token in text for token in ["₩", "KRW"]): - return "KRW" - if any(token in text for token in ["£", "GBP"]): - return "GBP" - if any(token in text for token in ["₽", "RUB"]): - return "RUB" - if any(token in text for token in ["AED", "د.إ"]): - return "AED" - if any(token in text for token in ["CA$", "CAD"]): - return "CAD" - - text = upper - if text in CURRENCY_ENUM_VALUES: - return text - return "USD" if "$" in str(value) else "USD" - - def _normalize_drive(self, value: Any) -> str | None: - return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA") - - def _normalize_gearbox(self, value: Any) -> str | None: - return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA") - - def _normalize_steering(self, value: Any) -> str | None: - return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None) - - def _normalize_body_type(self, value: Any) -> str: - return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" - - def _normalize_country(self, value: Any) -> str: - fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA" - return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback - - def _normalize_selling_type(self, value: Any) -> str: - text = self._as_str(value) or "AUCTION" - return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" - - def _map_value( - self, - value: Any, - mapping: dict[str, str], - allowed_values: tuple[str, ...], - *, - empty_default: str | None, - fallback: str | None, - ) -> str | None: - # Общий helper для enum-нормализации. - text = self._as_str(value).lower() - if not text: - return empty_default - if (mapped := mapping.get(text)) and mapped in allowed_values: - return mapped - for marker, mapped in mapping.items(): - if marker in text and mapped in allowed_values: - return mapped - return fallback - - @staticmethod - def _normalize_color(value: Any) -> str: - text = str(value).strip() if value is not None else "other" - if not text: - return "other" - if "/" in text: - text = text.split("/")[0].strip() - return text.lower() or "other" - - @staticmethod - def _boolish(value: Any) -> bool: - return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"} - - def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool: - for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]: - text = self._as_str(value).lower() - if text and text not in self.NO_DAMAGE_MARKERS: - return True - return False - - def _bool_sold(self, auction: dict[str, Any]) -> bool: - return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) - - def _build_images(self, urls: list[Any]) -> list[ImageRecord]: - # Для imageKeys берем самый большой размер. - best_by_key: dict[str, str] = {} - key_order: list[str] = [] - non_keyed: list[str] = [] - for item in urls: - if not isinstance(item, str): - continue - url = item.strip() - if not url: - continue - if m := re.search(r'imageKeys=([^&]+)', url): - img_key = m.group(1) - w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0 - existing = best_by_key.get(img_key) - if existing is None: - best_by_key[img_key] = url - key_order.append(img_key) - else: - existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0 - if w > existing_w: - best_by_key[img_key] = url - elif url not in non_keyed: - non_keyed.append(url) - deduped = [best_by_key[k] for k in key_order] + non_keyed - return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)] - - @staticmethod - def _make_fullres_url(url: str) -> str: - if "vis.iaai.com" in url: - return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) - return url - - @staticmethod - def _make_preview_url(url: str) -> str: - if "vis.iaai.com" in url: - preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) - if preview != url: - return preview - return url - - # Формирование parser_id. - - _PARSER_ID_ALPHABET = ascii_letters + digits - - @classmethod - def _generate_parser_id(cls, origin_id: str) -> str: - # Стабильный parser_id по origin_id. - digest = hashlib.sha256(origin_id.encode()).digest() - alphabet = cls._PARSER_ID_ALPHABET - base = len(alphabet) - num = int.from_bytes(digest[:17], "big") # 17 bytes = 136 bits, enough for 22 chars - chars: list[str] = [] - for _ in range(22): - num, idx = divmod(num, base) - chars.append(alphabet[idx]) - return "car-" + "".join(chars) - - def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: - # Формат: iaai:{lot_number} (аналог copart:94323985). - for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]: - text = self._as_str(value) - if text: - return f"iaai:{text}" - tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] - if "~" in tail: - tail = tail.split("~")[0] - raw = tail or self._slugify(vehicle_url) - return f"iaai:{raw}" - - @staticmethod - def _slugify(value: str) -> str: - return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car" - - diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py deleted file mode 100644 index 5a3c244..0000000 --- a/iaai_scraper/parsing/parser.py +++ /dev/null @@ -1,408 +0,0 @@ -import html as html_module -import json -import logging -import re -from typing import Any - -from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty - -logger = logging.getLogger("iaai_scraper.parsers") - - -class VehicleParser: - # Парсер данных страницы автомобиля. - - # Регулярные выражения для парсинга и детекции защиты. - _BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE) - _CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]) - _ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"]) - _CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE) - _ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE) - - SUMMARY_KEY_MAP = { - "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, - "year": {"year"}, - "make": {"make", "manufacturer", "brand"}, - "model": {"model"}, - "trim": {"trim", "series"}, - "odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"}, - "primary_damage": {"primarydamage", "damage", "damagetype", "loss"}, - "secondary_damage": {"secondarydamage"}, - "run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"}, - "buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"}, - "current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"}, - "actual_cash_value": {"actualcashvalue", "acv"}, - "estimated_repair_cost": {"estimatedrepaircost", "repaircost"}, - "keys": {"keys", "keystatus"}, - "title": {"titletype", "title", "documenttype"}, - "seller": {"seller", "sellername"}, - "location": {"location", "branchname", "auctionlocation", "branch"}, - "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, - "body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"}, - "drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"}, - "gearbox": {"transmission", "gearbox", "transmissiontype"}, - "engine": {"engine", "enginevolume", "enginetype", "enginedescription"}, - "fuel_type": {"fueltype", "fuel"}, - "cylinders": {"cylinders", "cylindercount"}, - "color": {"color", "primarycolor", "exteriorcolor"}, - } - - DOM_LABEL_MAP: dict[str, str] = { - "stock #": "lot_number", - "stock": "lot_number", - "primary damage": "primary_damage", - "secondary damage": "secondary_damage", - "odometer": "odometer", - "odometer (miles)": "odometer", - "mileage": "odometer", - "body style": "body_type", - "body type": "body_type", - "vehicle type": "body_type", - "engine": "engine", - "engine type": "engine", - "transmission": "gearbox", - "drive line type": "drive", - "driveline type": "drive", - "drive line": "drive", - "driveline": "drive", - "drive type": "drive", - "fuel type": "fuel_type", - "fuel": "fuel_type", - "cylinders": "cylinders", - "exterior/interior": "color", - "exterior color": "color", - "color": "color", - "model": "model", - "series": "trim", - "selling branch": "location", - "vehicle location": "vehicle_location", - "auction date and time": "auction_date", - "sale date": "auction_date", - "lane/run #": "lane", - "actual cash value": "actual_cash_value", - "estimated repair cost": "estimated_repair_cost", - "seller": "seller", - "title/sale doc": "title", - "title/sale doc brand": "title_brand", - "start code": "run_and_drive", - "key": "keys", - "keys": "keys", - "manufactured in": "manufactured_in", - "vehicle class": "vehicle_class", - } - - def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]: - result: dict[str, str] = {} - if not dom_text: - return result - lines = [line.strip() for line in dom_text.split("\n") if line.strip()] - known_labels = set(self.DOM_LABEL_MAP.keys()) - skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"} - max_fields = len(set(self.DOM_LABEL_MAP.values())) - - for i, line in enumerate(lines): - # Ранний выход, когда уже нашли все поля. - if len(result) >= max_fields: - break - - # Сценарий 1: "метка: значение" в одной строке. - colon_pos = line.find(":") - if colon_pos > 0: - label_part = line[:colon_pos].strip().lower() - value_part = line[colon_pos + 1:].strip() - if label_part in known_labels and value_part and value_part.lower() not in skip_values: - field_name = self.DOM_LABEL_MAP[label_part] - if field_name not in result or not result[field_name]: - result[field_name] = value_part - continue - - # Сценарий 2: метка и значение на соседних строках. - clean = line.rstrip(":").strip().lower() - clean_alt = clean.rstrip("#").strip() - matched_label = None - if clean in known_labels: - matched_label = clean - elif clean_alt in known_labels: - matched_label = clean_alt - - if matched_label and i + 1 < len(lines): - value = lines[i + 1].strip() - if value.rstrip(":").lower().strip() in known_labels: - continue - if value.lower() in skip_values: - continue - field_name = self.DOM_LABEL_MAP[matched_label] - if field_name not in result or not result[field_name]: - result[field_name] = value - return result - - def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]: - result: dict[str, str | None] = {"year": None, "make": None, "model": None} - title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "") - if title_match: - result["year"] = title_match.group(1) - result["make"] = title_match.group(2) - result["model"] = title_match.group(3) - return result - dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "") - if dom_match: - result["year"] = dom_match.group(1) - result["make"] = dom_match.group(2) - result["model"] = dom_match.group(3).strip() - return result - - def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]: - page_html = page_html or "" - dom_text = dom_text or "" - network_dump = network_dump or {} - responses = network_dump.get("json_responses", []) - payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))] - dom_kv = self._parse_dom_key_value_pairs(dom_text) - page_title = "" - title_match = re.search(r"]*>(.*?)", page_html or "", re.IGNORECASE | re.DOTALL) - if title_match: - page_title = title_match.group(1).strip() - title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) - - # Один проход по payload для всех полей. - all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP) - - summary: dict[str, Any] = {"source_url": vehicle_url} - for field, values in all_found.items(): - if field in dom_kv: - values.append(dom_kv[field]) - summary[field] = first_non_empty(values) - - summary["year"] = summary.get("year") or title_parsed.get("year") - summary["make"] = summary.get("make") or title_parsed.get("make") - summary["model"] = summary.get("model") or title_parsed.get("model") - summary["trim"] = summary.get("trim") or dom_kv.get("trim") - summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) - summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) - for dom_field, dom_value in dom_kv.items(): - if dom_field not in summary or not summary[dom_field]: - summary[dom_field] = dom_value - prices = self._extract_prices_from_text(dom_text) - if not summary.get("actual_cash_value") and prices: - summary["actual_cash_value"] = prices[0] - if not summary.get("buy_now"): - buy_now_match = self._BUY_NOW_RE.search(dom_text or "") - if buy_now_match: - summary["buy_now"] = buy_now_match.group(1) - elif prices: - summary["buy_now"] = prices[0] - if not summary.get("current_bid") and len(prices) > 1: - summary["current_bid"] = prices[1] - - embedded = self._extract_embedded_json(page_html) - for item in embedded: - p = item.get("payload") - if isinstance(p, (dict, list)): - payloads.append(p) - # Дополнительный проход по встроенному JSON. - extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP) - for field, vals in extra.items(): - if not summary.get(field): - v = first_non_empty(vals) - if v: - summary[field] = v - - # Передаём image_urls без повторного извлечения. - image_urls = summary.get("image_urls") or [] - return { - "vehicle_summary": summary, - "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls), - "embedded_json": embedded, - "dom_hints": self._dom_hints(dom_text), - "access_notes": self._build_access_notes(summary, responses), - } - - def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]: - if image_urls is None: - image_urls = self._extract_image_urls(payloads, "", vehicle_url) - return { - "vehicle_core": { - "lot_number": summary.get("lot_number"), "year": summary.get("year"), - "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), - "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), - "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), - "body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"), - "engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"), - "color": summary.get("color"), "keys": summary.get("keys"), - }, - "pricing": { - "buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"), - "actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"), - "currency": self._guess_currency(summary), - }, - "bids": self._build_bid_insights(summary, payloads), - "damage": { - "primary": summary.get("primary_damage"), - "secondary": summary.get("secondary_damage"), - "description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})), - }, - "auction": { - "auction_date": summary.get("auction_date"), - "lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})), - "branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]), - "sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})), - "item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]), - }, - "images": {"count": len(image_urls), "urls": image_urls}, - "source_endpoints": self._build_source_endpoints(responses), - } - - def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]: - return { - "amount": summary.get("current_bid"), - "currency": self._guess_currency(summary), - "bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})), - "status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})), - } - - def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]: - mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []} - for item in responses: - url = item.get("url", "") - category = item.get("category", "other") - if category == "vehicle": - mapping["vehicle"].append(url) - lowered = url.lower() - if any(token in lowered for token in ["bid", "offer"]): - mapping["bids"].append(url) - if any(token in lowered for token in ["damage", "report"]): - mapping["damage"].append(url) - if any(token in lowered for token in ["auction", "sale", "lane", "branch"]): - mapping["auction"].append(url) - elif category in mapping: - mapping[category].append(url) - return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()} - - def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: - endpoints = [item.get("url", "") for item in responses] - dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None)) - return { - "images_visible": bool(summary.get("image_urls")), - "network_json_count": len(responses), - "possible_captcha": bool(dom_hints.get("has_captcha_text")), - "possible_antibot": bool(dom_hints.get("has_antibot_text")), - "observed_endpoints": endpoints[:20], - } - - @staticmethod - def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]: - lowered = {key.lower() for key in keys} - values: list[Any] = [] - for payload in payloads: - values.extend(deep_find_key(payload, lowered)) - return values - - @staticmethod - def _guess_currency(summary: dict[str, Any]) -> str: - for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]: - value = str(summary.get(key) or "") - if "$" in value: - return "USD" - if "€" in value: - return "EUR" - if "¥" in value: - return "JPY" - return "USD" - - @staticmethod - def _extract_lot_number(text: str) -> str | None: - match = LOT_RE.search(text or "") - return match.group(1) if match else None - - @staticmethod - def _extract_prices_from_text(text: str) -> list[str]: - return [match.group(1) for match in PRICE_RE.finditer(text or "")] - - @staticmethod - def _extract_embedded_json(html: str) -> list[dict[str, Any]]: - scripts = re.findall(r"]*>(.*?)", html or "", flags=re.DOTALL | re.IGNORECASE) - extracted: list[dict[str, Any]] = [] - for script_text in scripts: - if "{" not in script_text and "[" not in script_text: - continue - # Пропускаем слишком большие минифицированные блоки. - if len(script_text) > 51_200: - continue - try: - parsed = json.loads(script_text.strip()) - except Exception: - continue - extracted.append({"type": "inline_json", "payload": parsed}) - return extracted - - @staticmethod - def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]: - vehicle_key = "" - key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "") - if key_match: - vehicle_key = key_match.group(1) - found: list[str] = [] - for payload in payloads: - found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"})) - flat: list[str] = [] - seen_flat: set[str] = set() - for item in found: - if isinstance(item, str) and item.startswith("http"): - cleaned = html_module.unescape(item) - lowered = cleaned.lower() - if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: - continue - if cleaned not in seen_flat: - seen_flat.add(cleaned) - flat.append(cleaned) - elif isinstance(item, list): - for child in item: - if isinstance(child, str) and child.startswith("http"): - cleaned = html_module.unescape(child) - lowered = cleaned.lower() - if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: - continue - if cleaned not in seen_flat: - seen_flat.add(cleaned) - flat.append(cleaned) - for pattern in [r']+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']: - for match in re.finditer(pattern, html or "", re.IGNORECASE): - url = html_module.unescape(match.group(1).strip()) - if not url.startswith("http") or url in seen_flat: - continue - lowered = url.lower() - if vehicle_key and vehicle_key in url: - seen_flat.add(url) - flat.append(url) - elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]): - if vehicle_key and vehicle_key not in url: - continue - seen_flat.add(url) - flat.append(url) - if vehicle_key: - for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""): - cleaned = html_module.unescape(url) - if cleaned not in seen_flat and vehicle_key in cleaned: - seen_flat.add(cleaned) - flat.append(cleaned) - filtered: list[str] = [] - for url in flat: - lowered = url.lower() - if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): - continue - if "vis.iaai.com" in lowered and "/resizer" not in lowered: - continue - filtered.append(url) - return filtered - - @staticmethod - def _dom_hints(text: str) -> dict[str, Any]: - lowered = (text or "").lower() - return { - "has_buy_now_text": "buy now" in lowered, - "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, - "has_damage_text": "damage" in lowered, - "has_title_text": "title" in lowered, - "has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS), - "has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS), - } diff --git a/iaai_scraper/proxy_bridge.py b/iaai_scraper/proxy_bridge.py deleted file mode 100644 index 251ce58..0000000 --- a/iaai_scraper/proxy_bridge.py +++ /dev/null @@ -1,317 +0,0 @@ -from __future__ import annotations - -import logging -import os -import select -import socket -import socketserver -import struct -import threading -from urllib.parse import urlsplit - -BUFFER_SIZE = 65536 -CRLF = b"\r\n" -DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1") -DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899")) -SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "") -SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002")) -SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "") -SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "") -RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60")) -MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64")) - -logger = logging.getLogger("proxy_bridge") - - -class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - allow_reuse_address = True - daemon_threads = True - - def __init__(self, server_address, request_handler_class): - super().__init__(server_address, request_handler_class) - self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS) - - def process_request_thread(self, request, client_address): - with self._worker_semaphore: - super().process_request_thread(request, client_address) - - -def _recv_exact(sock: socket.socket, size: int) -> bytes: - data = b"" - while len(data) < size: - chunk = sock.recv(size - len(data)) - if not chunk: - raise ConnectionError("Unexpected EOF from SOCKS5 server") - data += chunk - return data - - -def _socks5_connect(host: str, port: int) -> socket.socket: - if not SOCKS5_HOST: - raise RuntimeError("SOCKS5_PROXY_HOST is not configured") - - upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30) - upstream.settimeout(30) - - methods = [0x00] - if SOCKS5_USER or SOCKS5_PASS: - methods = [0x02] - upstream.sendall(bytes([0x05, len(methods), *methods])) - version, method = _recv_exact(upstream, 2) - if version != 0x05 or method == 0xFF: - upstream.close() - raise ConnectionError("SOCKS5 authentication negotiation failed") - - if method == 0x02: - username = SOCKS5_USER.encode("utf-8") - password = SOCKS5_PASS.encode("utf-8") - if len(username) > 255 or len(password) > 255: - upstream.close() - raise ValueError("SOCKS5 username/password too long") - upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password) - auth_version, auth_status = _recv_exact(upstream, 2) - if auth_version != 0x01 or auth_status != 0x00: - upstream.close() - raise ConnectionError("SOCKS5 username/password authentication failed") - - try: - socket.inet_aton(host) - addr_type = 0x01 - addr_payload = socket.inet_aton(host) - except OSError: - host_bytes = host.encode("idna") - if len(host_bytes) > 255: - upstream.close() - raise ValueError("Target host is too long for SOCKS5 domain format") - addr_type = 0x03 - addr_payload = bytes([len(host_bytes)]) + host_bytes - - request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port) - upstream.sendall(request) - - response_head = _recv_exact(upstream, 4) - version, reply, _reserved, reply_addr_type = response_head - if version != 0x05 or reply != 0x00: - upstream.close() - raise ConnectionError(f"SOCKS5 connect failed with code {reply}") - - if reply_addr_type == 0x01: - _recv_exact(upstream, 4) - elif reply_addr_type == 0x03: - domain_len = _recv_exact(upstream, 1)[0] - _recv_exact(upstream, domain_len) - elif reply_addr_type == 0x04: - _recv_exact(upstream, 16) - _recv_exact(upstream, 2) - - upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) - return upstream - - -def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None: - sockets = [left, right] - left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) - right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS) - try: - while True: - readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS) - if exceptional: - break - if not readable: - logger.debug("Relay idle timeout reached; closing sockets") - return - for current in readable: - other = right if current is left else left - data = current.recv(BUFFER_SIZE) - if not data: - return - other.sendall(data) - finally: - for sock in sockets: - try: - sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - try: - sock.close() - except OSError: - pass - - -class ProxyHandler(socketserver.StreamRequestHandler): - def handle(self) -> None: - try: - request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip() - if not request_line: - return - - method, target, version = request_line.split() - headers = self._read_headers() - logger.info("%s %s", method, target) - - if method.upper() == "CONNECT": - host, port = self._parse_connect_target(target) - logger.info("CONNECT %s:%s", host, port) - upstream = _socks5_connect(host, port) - self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF) - self.wfile.flush() - _relay_bidirectional(self.connection, upstream) - return - - host, port, path = self._parse_forward_target(target, headers) - upstream = _socks5_connect(host, port) - self._send_forward_request(upstream, method, path, version, headers) - body = self._read_request_body(headers) - if body: - upstream.sendall(body) - _relay_bidirectional(self.connection, upstream) - except Exception as exc: - logger.exception("Proxy bridge request failed: %s", exc) - try: - self.wfile.write( - b"HTTP/1.1 502 Bad Gateway" + CRLF - + b"Connection: close" + CRLF - + b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF - + b"Bad Gateway" - ) - self.wfile.flush() - except OSError: - pass - - def _read_headers(self) -> list[tuple[str, str]]: - headers: list[tuple[str, str]] = [] - while True: - line = self.rfile.readline(BUFFER_SIZE) - if line in {CRLF, b"\n", b""}: - break - decoded = line.decode("iso-8859-1") - if ":" not in decoded: - continue - name, value = decoded.split(":", 1) - headers.append((name.strip(), value.strip())) - return headers - - @staticmethod - def _parse_connect_target(target: str) -> tuple[str, int]: - if target.startswith("["): - end = target.find("]") - if end == -1 or len(target) <= end + 2 or target[end + 1] != ":": - raise ValueError("Invalid CONNECT target") - host = target[1:end] - port_text = target[end + 2 :] - return host, int(port_text) - - host, port_text = target.rsplit(":", 1) - return host, int(port_text) - - @staticmethod - def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]: - if target.startswith("http://"): - parts = urlsplit(target) - port = parts.port or 80 - path = parts.path or "/" - if parts.query: - path += f"?{parts.query}" - return parts.hostname or "", port, path - - if target.startswith("https://"): - raise ValueError("HTTPS absolute-form request must use CONNECT") - - host_header = next((value for name, value in headers if name.lower() == "host"), "") - if not host_header: - raise ValueError("Missing Host header") - if ":" in host_header: - host, port_text = host_header.rsplit(":", 1) - return host, int(port_text), target - return host_header, 80, target - - def _send_forward_request( - self, - upstream: socket.socket, - method: str, - path: str, - version: str, - headers: list[tuple[str, str]], - ) -> None: - filtered_headers: list[tuple[str, str]] = [] - hop_by_hop = { - "proxy-connection", - "proxy-authorization", - "connection", - "keep-alive", - "te", - "trailer", - "transfer-encoding", - "upgrade", - } - for name, value in headers: - if name.lower() in hop_by_hop: - continue - filtered_headers.append((name, value)) - - request_head = [f"{method} {path} {version}\r\n"] - request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers) - request_head.append("\r\n") - upstream.sendall("".join(request_head).encode("iso-8859-1")) - - def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes: - transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "") - if "chunked" in transfer_encoding.lower(): - return self._read_chunked_request_body() - - content_length = next((value for name, value in headers if name.lower() == "content-length"), None) - if not content_length: - return b"" - return self.rfile.read(int(content_length)) - - def _read_chunked_request_body(self) -> bytes: - chunks: list[bytes] = [] - while True: - size_line = self.rfile.readline(BUFFER_SIZE) - if not size_line: - raise ConnectionError("Unexpected EOF in chunked request") - size_text = size_line.strip().split(b";", 1)[0] - chunk_size = int(size_text, 16) - chunks.append(size_line) - if chunk_size == 0: - while True: - trailer_line = self.rfile.readline(BUFFER_SIZE) - if not trailer_line: - raise ConnectionError("Unexpected EOF in chunked trailers") - chunks.append(trailer_line) - if trailer_line in {CRLF, b"\n"}: - return b"".join(chunks) - - chunk_data = self.rfile.read(chunk_size) - if len(chunk_data) != chunk_size: - raise ConnectionError("Unexpected EOF in chunk body") - chunks.append(chunk_data) - chunk_end = self.rfile.read(2) - if chunk_end != CRLF: - raise ConnectionError("Invalid chunk terminator") - chunks.append(chunk_end) - - -def main() -> None: - if not SOCKS5_HOST: - raise SystemExit("SOCKS5_PROXY_HOST is required") - - logging.basicConfig( - level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(), - format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s", - ) - - with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server: - logger.info( - "Listening on %s:%s -> socks5://%s:%s (max_workers=%s)", - DEFAULT_LISTEN_HOST, - DEFAULT_LISTEN_PORT, - SOCKS5_HOST, - SOCKS5_PORT, - MAX_WORKERS, - ) - server.serve_forever() - - -if __name__ == "__main__": - main() diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py deleted file mode 100644 index 21c3778..0000000 --- a/iaai_scraper/scraper.py +++ /dev/null @@ -1,1360 +0,0 @@ -import json -import logging -import os -import re -import signal -import time -import uuid -import html as html_module -from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime, timezone -from pathlib import Path -from urllib.parse import urlsplit, urlunsplit -from urllib.request import Request, urlopen - -import urllib3 - -from playwright.sync_api import Error as PlaywrightError -from playwright.sync_api import BrowserContext, Page, sync_playwright -from playwright.sync_api import TimeoutError as PlaywrightTimeoutError - -from .browser import BrowserFactory, HumanPacer, NetworkCapture -from .core.config import Settings, settings -from .core.exceptions import AntiBotDetectedError, SiteStructureChangedError -from .core.logs import set_trace_id, setup_logging -from .core.retry import retryable -from .core.runtime_config import RuntimeConfig -from .core.utils import save_to_json -from .parsing.mapper import CarMapper -from .parsing.parser import VehicleParser -from .storage.db import PersistenceService -from .browser.listing import ListingCollector -from .storage.schemas import CarRecord - -logger = logging.getLogger("iaai_scraper.scraper") -VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) -HTML_TAG_RE = re.compile(r"<[^>]+>") -SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) - - -class IAAIScraper: - - @staticmethod - def _is_protection_or_network_error(exc: Exception) -> bool: - message = str(exc).lower() - signals = ( - "captcha", - "antibot", - "blocked", - "challenge", - "ns_error_net_interrupt", - "navigation", - "timeout", - "403", - "429", - ) - return any(signal in message for signal in signals) - - @staticmethod - def _html_to_text(html: str) -> str: - if not html: - return "" - cleaned = SCRIPT_STYLE_RE.sub(" ", html) - text = HTML_TAG_RE.sub(" ", cleaned) - text = html_module.unescape(text) - return re.sub(r"\s+", " ", text).strip() - - @staticmethod - def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None: - dom_hints = parsed.get("dom_hints", {}) or {} - access_notes = parsed.get("access_notes", {}) or {} - summary = parsed.get("vehicle_summary", {}) or {} - - has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model")) - - if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity: - raise AntiBotDetectedError(f"IAAI anti-bot detected for {vehicle_url}") - - if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity: - raise AntiBotDetectedError(f"IAAI blocked or challenged request for {vehicle_url}") - - if not has_identity: - raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") - - @staticmethod - def _extract_origin_id_from_url(vehicle_url: str) -> str | None: - match = VEHICLE_ID_RE.search(vehicle_url) - if not match: - return None - return match.group(1) - - @staticmethod - def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None: - raw_id = IAAIScraper._extract_origin_id_from_url(vehicle_url) - if not raw_id: - return None - return f"iaai:{raw_id}" - - @staticmethod - def _normalize_vehicle_url(vehicle_url: str) -> str: - try: - parts = urlsplit(vehicle_url) - return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) - except Exception: - return vehicle_url - - def __init__(self, runtime_settings: Settings | None = None) -> None: - self.settings = runtime_settings or settings - setup_logging(self.settings.log_level, self.settings.log_file) - self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) - self.trace_id = uuid.uuid4().hex[:12] - set_trace_id(self.trace_id) - self.playwright = None - self.browser = None - self.context: BrowserContext | None = None - self.browser_factory = BrowserFactory(self.settings) - self.pacer = HumanPacer(self.settings) - self.listing_collector = ListingCollector(self.settings, self.pacer) - self.vehicle_parser = VehicleParser() - self.car_mapper = CarMapper() - self.persistence = PersistenceService(self.settings) - self._shutdown_requested = False - # HTTP-клиент для fast-path. - # При наличии прокси используем ProxyManager. - proxy_url = self.settings.proxy.server - if proxy_url: - _proxy_kwargs: dict = { - "num_pools": 4, - "maxsize": 64, - "retries": False, - "timeout": urllib3.Timeout(connect=5, read=10), - } - if self.settings.proxy.username: - _proxy_kwargs["proxy_headers"] = urllib3.make_headers( - proxy_basic_auth=f"{self.settings.proxy.username}:{self.settings.proxy.password or ''}" - ) - self._http_pool: urllib3.PoolManager = urllib3.ProxyManager(proxy_url, **_proxy_kwargs) - logger.info("HTTP fast-path using proxy: %s", proxy_url) - else: - self._http_pool = urllib3.PoolManager( - num_pools=4, - maxsize=64, - retries=False, - timeout=urllib3.Timeout(connect=5, read=10), - ) - - def _new_trace_id(self, prefix: str) -> str: - trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" - self.trace_id = trace_id - set_trace_id(trace_id) - return trace_id - - def __enter__(self) -> "IAAIScraper": - if self.playwright is None: - self.playwright = sync_playwright().start() - if self.browser is None: - self.browser = self.browser_factory.create_browser(self.playwright) - return self - - def __exit__(self, exc_type, exc, tb) -> None: - self.close() - - def close(self) -> None: - if self.context is not None: - try: - self.context.close() - except PlaywrightError: - pass - finally: - self.context = None - if self.browser is not None: - try: - self.browser.close() - except PlaywrightError: - pass - finally: - self.browser = None - if self.playwright is not None: - try: - self.playwright.stop() - except PlaywrightError: - pass - finally: - self.playwright = None - - def _new_context(self) -> BrowserContext: - if self.browser is None: - self.__enter__() - if self.context: - try: - self.context.close() - except PlaywrightError: - pass - self.context = self.browser_factory.create_context(self.browser) - return self.context - - def init_db(self): - self.persistence.create_tables() - return {"status": "ok", "database_url": self.settings.database.url} - - def _warmup_visit(self, page: Page) -> None: - # Прогрев главной страницы. - try: - logger.info("Warmup: visiting homepage to pass anti-bot challenge...") - page.goto(self.settings.home_url, wait_until="commit", timeout=30_000) - # Ждём domcontentloaded вместо networkidle. - try: - page.wait_for_load_state("domcontentloaded", timeout=6_000) - except PlaywrightTimeoutError: - pass - # Короткая проверка, что страница стабилизировалась. - try: - page.wait_for_function("() => document.title && document.title.length > 3", timeout=4_000) - except PlaywrightTimeoutError: - pass - # Короткая пауза для установки cookies. - time.sleep(0.3) - logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50]) - except Exception as e: - logger.warning("Warmup visit failed: %s — continuing anyway", e) - time.sleep(1.5) - - def _get_page_with_warmup(self) -> Page: - context = self._new_context() - page = context.new_page() - # Прогрев с первой антибот-проверкой. - self._warmup_visit(page) - return page - - def _dedupe_urls(self, raw_urls: list[str]) -> list[str]: - # Нормализация и дедупликация URL. - vehicle_urls: list[str] = [] - seen: set[str] = set() - for raw in raw_urls: - normalized = self._normalize_vehicle_url(raw) - if normalized not in seen: - seen.add(normalized) - vehicle_urls.append(normalized) - return vehicle_urls - - def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: - # Отсев уже известных URL. - url_to_origin_id = { - url: self._extract_db_origin_id_from_url(url) - for url in vehicle_urls - } - candidate_origin_ids = [oid for oid in url_to_origin_id.values() if oid] - existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( - vehicle_urls, candidate_origin_ids, - ) - known_urls = { - url for url in vehicle_urls - if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) - } - skipped = len(known_urls) - if skipped: - logger.info("Filtering already known vehicles: skipped %d", skipped) - new_urls = [url for url in vehicle_urls if url not in known_urls] - return new_urls, skipped - - def _collect_listing_iterative( - self, - *, - make: str | None = None, - model: str | None = None, - limit: int, - ) -> tuple[list[str], list[str], dict, int]: - # Поэтапный сбор листинга до нужного лимита новых URL. - new_urls: list[str] = [] - all_raw_urls: list[str] = [] - seen: set[str] = set() - skipped_existing = 0 - pages_info: list[dict] = [] - max_pages = self.settings.listing.max_pages_per_run - - page = self._get_page_with_warmup() - try: - self.listing_collector.open_cars_listing(page) - self.listing_collector.apply_filters(page, make=make, model=model) - - for page_number in range(1, max_pages + 1): - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - pages_info.append({ - "page_number": page_result.page_number, - "links_found": len(page_result.vehicle_links), - }) - - # Собираем URL со страницы. - page_urls: list[str] = [] - for item in page_result.vehicle_links: - normalized = self._normalize_vehicle_url(item.href) - all_raw_urls.append(item.href) - if normalized not in seen: - seen.add(normalized) - page_urls.append(normalized) - - if not page_urls: - # Страница 1 пустая — скорее всего transient network issue. - # Пробуем перезагрузить листинг ещё раз. - if page_number == 1: - logger.warning("Page 1 returned 0 links — retrying listing open...") - time.sleep(3) - self.listing_collector.open_cars_listing(page) - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - for item in page_result.vehicle_links: - normalized = self._normalize_vehicle_url(item.href) - all_raw_urls.append(item.href) - if normalized not in seen: - seen.add(normalized) - page_urls.append(normalized) - if not page_urls: - logger.info("Page %d: 0 new links, stopping pagination", page_number) - break - - # Фильтруем known. - fresh, page_skipped = self._filter_known_urls(page_urls) - skipped_existing += page_skipped - new_urls.extend(fresh) - - logger.info( - "Page %d: %d links, %d new, %d known (total new: %d/%d)", - page_number, len(page_urls), len(fresh), page_skipped, - len(new_urls), limit, - ) - - # Стопаем только если набрали нужное количество И на этой странице уже нет новых. - # Если последняя страница дала новые — проверяем следующую (там могут быть ещё). - if len(new_urls) >= limit and len(fresh) == 0: - break - if len(new_urls) >= limit: - # Набрали достаточно, дальше не листаем - break - - if not page_result.next_page_detected: - logger.info("No next page detected, stopping") - break - if not self.listing_collector.go_to_next_page(page): - logger.info("Failed to navigate to next page, stopping") - break - finally: - page.close() - - # Обрезаем до limit. - new_urls = new_urls[:limit] - - listing = { - "status": "ok", - "listing_url": self.settings.listing.cars_url, - "pages_collected": len(pages_info), - "vehicles_collected": len(all_raw_urls), - "vehicle_urls": all_raw_urls, - "early_stopped": False, - "pages": pages_info, - } - return new_urls, all_raw_urls, listing, skipped_existing - - def collect_listing( - self, - make: str | None = None, - model: str | None = None, - known_origin_ids: set[str] | None = None, - ): - page = self._get_page_with_warmup() - - try: - listing = self.listing_collector.collect_listing_links( - page, - make=make, - model=model, - known_origin_ids=known_origin_ids, - ) - finally: - page.close() - - return { - "status": "ok", - **listing, - } - - def _get_page(self) -> Page: - if not self.context: - self._new_context() - return self.context.new_page() - - @retryable(max_attempts=3, jitter_seconds=0.25) - def scrape_vehicle_detail(self, vehicle_url: str): - # Открыть страницу, перехватить JSON, вернуть данные. - page = self._get_page() - try: - return self._scrape_on_page(page, vehicle_url) - finally: - page.close() - - # Быстрый извлекатель данных из inline JSON. - _JS_EXTRACT = """ - () => { - try { - const scripts = document.querySelectorAll('script:not([src])'); - for (const s of scripts) { - const t = s.textContent || ''; - if (!t.includes('inventoryView') || !t.includes('attributes')) continue; - const start = t.indexOf('{'); - if (start < 0) continue; - let depth = 0, end = -1; - for (let i = start; i < t.length; i++) { - if (t[i] === '{') depth++; - else if (t[i] === '}') { depth--; if (depth === 0) { end = i; break; } } - } - if (end < 0) continue; - try { - const obj = JSON.parse(t.slice(start, end + 1)); - const iv = obj.inventoryView; - if (!iv || !iv.attributes) continue; - const attr = iv.attributes; - const imgs = (iv.imageDimensions && iv.imageDimensions.keys && iv.imageDimensions.keys.$values) - ? iv.imageDimensions.keys.$values : []; - const bid = (obj.auctionInformation && obj.auctionInformation.biddingInformation) - ? obj.auctionInformation.biddingInformation : {}; - const prebid = (obj.auctionInformation && obj.auctionInformation.prebidInformation) - ? obj.auctionInformation.prebidInformation : {}; - return { - ok: true, - SalvageId: attr.SalvageId || '', - StockNumber: attr.StockNumber || '', - Year: attr.Year || '', - Make: attr.Make || '', - Model: attr.Model || '', - Series: attr.Series || '', - BodyStyleName: attr.BodyStyleName || '', - Cylinders: attr.Cylinders || '', - DriveLineTypeDesc: attr.DriveLineTypeDesc || '', - EngineSize: (attr.EngineInformation || attr.EngineSize || '').trim(), - FuelTypeCode: attr.FuelTypeCode || '', - Transmission: attr.Transmission || '', - ExteriorColor: attr.ExteriorColor || '', - PrimaryDamageDesc: attr.PrimaryDamageDesc || '', - SecondaryDamageDesc: attr.SecondaryDamageDesc || '', - ODOValue: attr.ODOValue || '', - ODOBrand: attr.ODOBrand || '', - RunAndDrive: attr.RunAndDrive || '', - Keys: attr.Keys || '', - BranchName: attr.BranchName || '', - AuctionDateTime: attr.AuctionDateTime || '', - Title: attr.Title || '', - TitleBrand: attr.TitleBrand || '', - TitleCode: attr.TitleCode || '', - EstRepairCost: attr.EstRepairCost || '', - VehicleGrade: attr.VehicleGrade || '', - highBidAmount: prebid.highBidAmount || bid.highBidAmount || '', - buyNowPrice: prebid.buyNowPrice || bid.buyNowPrice || '', - acv: attr.ProviderACV || '', - BranchNumber: attr.BranchNumber || '', - imageKeys: imgs.map(i => i.k || '').filter(Boolean), - }; - } catch (_) { continue; } - } - } catch (_) {} - return { ok: false }; - } - """ - - @staticmethod - def _build_car_from_js(js_data: dict, vehicle_url: str) -> dict: - # Сбор vehicle_summary из JS-данных. - if not js_data or not js_data.get("ok"): - return {} - - brnch = str(js_data.get("BranchNumber", "") or "").strip() - img_keys = js_data.get("imageKeys") or [] - image_urls = [ - f"https://vis.iaai.com/resizer?imageKeys={k}&width=845&height=633" - for k in img_keys - ] - - return { - "source_url": vehicle_url, - "lot_number": js_data.get("StockNumber") or js_data.get("SalvageId"), - "year": js_data.get("Year"), - "make": js_data.get("Make"), - "model": js_data.get("Model"), - "trim": js_data.get("Series"), - "body_type": js_data.get("BodyStyleName"), - "cylinders": js_data.get("Cylinders"), - "drive": js_data.get("DriveLineTypeDesc"), - "engine": js_data.get("EngineSize"), - "fuel_type": js_data.get("FuelTypeCode"), - "gearbox": js_data.get("Transmission"), - "color": js_data.get("ExteriorColor"), - "primary_damage": js_data.get("PrimaryDamageDesc"), - "secondary_damage": js_data.get("SecondaryDamageDesc"), - "odometer": js_data.get("ODOValue"), - "run_and_drive": js_data.get("RunAndDrive"), - "keys": js_data.get("Keys"), - "location": js_data.get("BranchName"), - "auction_date": js_data.get("AuctionDateTime"), - "title": js_data.get("Title"), - "current_bid": js_data.get("highBidAmount"), - "buy_now": js_data.get("buyNowPrice"), - "actual_cash_value": js_data.get("acv"), - "estimated_repair_cost": js_data.get("EstRepairCost"), - "image_urls": image_urls, - } - - @staticmethod - def _build_payload_insights(vehicle_summary: dict) -> dict: - # Сбор payload_insights из vehicle_summary. - return { - "vehicle_core": vehicle_summary, - "pricing": { - "buy_now": vehicle_summary.get("buy_now"), - "current_bid": vehicle_summary.get("current_bid"), - "actual_cash_value": vehicle_summary.get("actual_cash_value"), - "estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"), - "currency": "USD", - }, - "bids": {"amount": vehicle_summary.get("current_bid"), "currency": "USD"}, - "damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")}, - "auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")}, - "images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []}, - } - - def _scrape_on_page(self, page: Page, vehicle_url: str): - trace_id = self._new_trace_id("scrape") - started_at = time.perf_counter() - - # Блокируем тяжёлые ресурсы страницы. - if self.settings.block_resources: - BrowserFactory.enable_resource_blocking(page) - - capture = NetworkCapture(self.settings) - capture.attach(page, origin_url=vehicle_url) - - page.goto(vehicle_url, wait_until="commit", timeout=60_000) - - # На VPS достаточно domcontentloaded. - try: - page.wait_for_load_state("domcontentloaded", timeout=5_000) - except PlaywrightTimeoutError: - pass - - # Быстрый путь: читаем данные из inline JSON. - js_data: dict = {} - try: - js_data = page.evaluate(self._JS_EXTRACT) or {} - except Exception: - pass - - if js_data.get("ok"): - # Успешное извлечение структуры авто. - vehicle_summary = self._build_car_from_js(js_data, vehicle_url) - # Быстрая проверка валидности данных. - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise SiteStructureChangedError(f"JS extraction returned no vehicle identity for {vehicle_url}") - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - network_dump = capture.export() - result = { - "trace_id": trace_id, - "source_url": vehicle_url, - "fetched_at_epoch": int(time.time()), - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "network": network_dump, - "vehicle_summary": vehicle_summary, - "payload_insights": {}, - "embedded_json": [], - "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, - "access_notes": {}, - "db_record": db_record.model_dump(mode="json"), - } - if self.settings.raw_output_json: - save_to_json(network_dump, self.settings.raw_output_json) - return result - - # Резервный путь: полный парсинг страницы. - try: - page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=400) - except PlaywrightTimeoutError: - pass - - html = page.content() - try: - dom_text = page.evaluate("() => document.body?.textContent || ''") - except Exception: - dom_text = "" - network_dump = capture.export() - parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) - self._raise_if_blocked_or_incomplete(parsed, vehicle_url) - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=parsed.get("vehicle_summary", {}), - payload_insights=parsed.get("payload_insights", {}), - ) - - result = { - "trace_id": trace_id, - "source_url": vehicle_url, - "fetched_at_epoch": int(time.time()), - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "network": network_dump, - **parsed, - "db_record": db_record.model_dump(mode="json"), - } - - if self.settings.raw_output_json: - save_to_json(network_dump, self.settings.raw_output_json) - return result - - @staticmethod - def _extract_iaai_json_from_html(html: str, vehicle_url: str) -> dict | None: - """Извлекает IAAI inventoryView.attributes из HTML без браузера. - - Использует json.JSONDecoder.raw_decode для быстрого поиска JSON - вместо посимвольного сканирования скобок. - """ - search = "inventoryView" - pos = html.find(search) - if pos < 0: - return None - - # Ищем начало внешнего JSON-объекта. - script_start = html.rfind("", script_start) - if tag_end < 0: - return None - content_start = html.find("{", tag_end) - if content_start < 0: - return None - - # Быстрый поиск конца JSON через raw_decode. - decoder = json.JSONDecoder() - try: - obj, _ = decoder.raw_decode(html, content_start) - except (json.JSONDecodeError, ValueError): - return None - - iv = obj.get("inventoryView") - if not iv or not iv.get("attributes"): - return None - - attr = iv["attributes"] - imgs = [] - try: - imgs = iv.get("imageDimensions", {}).get("keys", {}).get("$values", []) or [] - except Exception: - pass - - bid = {} - prebid = {} - try: - ai = obj.get("auctionInformation", {}) - bid = ai.get("biddingInformation", {}) or {} - prebid = ai.get("prebidInformation", {}) or {} - except Exception: - pass - - img_keys = [i.get("k", "") for i in imgs if i.get("k")] - return { - "ok": True, - "SalvageId": attr.get("SalvageId", ""), - "StockNumber": attr.get("StockNumber", ""), - "Year": attr.get("Year", ""), - "Make": attr.get("Make", ""), - "Model": attr.get("Model", ""), - "Series": attr.get("Series", ""), - "BodyStyleName": attr.get("BodyStyleName", ""), - "Cylinders": attr.get("Cylinders", ""), - "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), - "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), - "FuelTypeCode": attr.get("FuelTypeCode", ""), - "Transmission": attr.get("Transmission", ""), - "ExteriorColor": attr.get("ExteriorColor", ""), - "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), - "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), - "ODOValue": attr.get("ODOValue", ""), - "ODOBrand": attr.get("ODOBrand", ""), - "RunAndDrive": attr.get("RunAndDrive", ""), - "Keys": attr.get("Keys", ""), - "BranchName": attr.get("BranchName", ""), - "AuctionDateTime": attr.get("AuctionDateTime", ""), - "Title": attr.get("Title", ""), - "TitleBrand": attr.get("TitleBrand", ""), - "TitleCode": attr.get("TitleCode", ""), - "EstRepairCost": attr.get("EstRepairCost", ""), - "VehicleGrade": attr.get("VehicleGrade", ""), - "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), - "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), - "acv": attr.get("ProviderACV", ""), - "BranchNumber": attr.get("BranchNumber", ""), - "imageKeys": img_keys, - } - - def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord: - # Быстрый запрос через context.request. - if not self.context: - self._new_context() - assert self.context is not None - last_error: Exception | None = None - max_attempts = max(1, self.settings.fast_path_max_attempts) - timeout_ms = max(1000, self.settings.fast_path_timeout_ms) - for attempt in range(1, max_attempts + 1): - try: - response = self.context.request.get(vehicle_url, timeout=timeout_ms) - if not response.ok: - raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") - - html = response.text() - - # Пытаемся извлечь JSON напрямую из HTML. - js_data = self._extract_iaai_json_from_html(html, vehicle_url) - if not js_data: - raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") - - vehicle_summary = self._build_car_from_js(js_data, vehicle_url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - return CarRecord.model_validate(db_record.model_dump(mode="json")) - except Exception as exc: - last_error = exc - if attempt < max_attempts: - time.sleep(0.2) - - if last_error is not None: - raise last_error - raise RuntimeError(f"HTTP fast-path failed for {vehicle_url}") - - def _cookie_header_for_context(self) -> str: - if not self.context: - return "" - try: - cookies = self.context.cookies() - except Exception: - return "" - pairs = [ - f"{item.get('name')}={item.get('value')}" - for item in cookies - if item.get("name") and item.get("value") is not None - ] - return "; ".join(pairs) - - def _scrape_via_raw_http( - self, - vehicle_url: str, - *, - cookie_header: str, - user_agent: str, - ) -> CarRecord: - """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive).""" - headers = { - "User-Agent": user_agent, - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US,en;q=0.9", - "Cache-Control": "no-cache", - "Pragma": "no-cache", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1", - } - if cookie_header: - headers["Cookie"] = cookie_header - - response = self._http_pool.request("GET", vehicle_url, headers=headers) - if response.status >= 400: - raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") - html = response.data.decode("utf-8", errors="ignore") - - js_data = self._extract_iaai_json_from_html(html, vehicle_url) - if not js_data: - raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") - - vehicle_summary = self._build_car_from_js(js_data, vehicle_url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") - - db_record = self.car_mapper.map_to_car_record( - vehicle_url=vehicle_url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - return CarRecord.model_validate(db_record.model_dump(mode="json")) - - def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"): - # Скрапинг и upsert одного авто. - trace_id = self._new_trace_id("sync-vehicle") - started_at = time.perf_counter() - self.persistence.create_tables() - run_id = self.persistence.start_sync_run(lane=lane) - ids_fetched = 1 - cars_upserted = 0 - cars_failed = 0 - images_upserted = 0 - status = "failed" - error_summary = None - try: - scrape_result = self.scrape_vehicle_detail(vehicle_url) - db_record = scrape_result.get("db_record") - if not db_record: - raise RuntimeError("Scrape result does not contain db_record") - record = CarRecord.model_validate(db_record) - upsert = self.persistence.upsert_car(record) - cars_upserted = 1 - images_upserted = int(upsert.get("images_upserted", 0)) - status = "success" - return { - "trace_id": trace_id, - "status": status, - "run_id": run_id, - "vehicle_url": vehicle_url, - "db_action": upsert.get("action"), - "images_upserted": images_upserted, - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "db_record": db_record, - } - except Exception as exc: - cars_failed = 1 - status = "failed" - error_summary = str(exc) - raise - finally: - self.persistence.finish_sync_run( - run_id, - status=status, - ids_fetched=ids_fetched, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - images_upserted=images_upserted, - error_summary=error_summary, - ) - - # ── Tunables for sync_batch ── - _HTTP_MICRO_BATCH = 25 # URLs per micro-batch (avoid mass rate-limit) - _HTTP_MAX_RETRIES = 2 # Retries per URL before giving up to browser - _HTTP_RETRY_DELAYS = (0.4, 1.0) # Backoff between retries - _FALLBACK_PARALLEL_PAGES = 4 # Concurrent browser tabs for fallback - _FALLBACK_NAV_TIMEOUT_MS = 8000 # Reduced from 15 000 - - def _browser_fallback_parallel( - self, - fallback_urls: list[tuple[str, int]], - total: int, - n_pages: int, - ) -> dict: - # Обработка fallback URL через браузер. - records: list[CarRecord] = [] - failures: list[dict[str, str]] = [] - cars_failed = 0 - protection_events = 0 - - # Деление URL по страницам. - slices: list[list[tuple[str, int]]] = [[] for _ in range(n_pages)] - for i, item in enumerate(fallback_urls): - slices[i % n_pages].append(item) - - def _process_slice(url_slice: list[tuple[str, int]]) -> dict: - local_records: list[CarRecord] = [] - local_failures: list[dict[str, str]] = [] - local_failed = 0 - local_protection = 0 - page: Page | None = None - try: - for url, global_idx in url_slice: - if page is None: - page = self._get_page() - if self.settings.block_resources: - BrowserFactory.enable_resource_blocking(page) - - try: - page.goto( - url, - wait_until="commit", - timeout=self._FALLBACK_NAV_TIMEOUT_MS, - ) - except Exception as exc: - if self._is_protection_or_network_error(exc): - local_protection += 1 - local_failed += 1 - local_failures.append({"vehicle_url": url, "error": str(exc)}) - logger.error("[%d/%d] Failed to open %s: %s", global_idx, total, url, exc) - try: - page.close() - except Exception: - pass - page = None - continue - - try: - js_data: dict = {} - try: - js_data = page.evaluate(self._JS_EXTRACT) or {} - except Exception: - pass - - if not js_data.get("ok"): - try: - page.wait_for_load_state("domcontentloaded", timeout=3_000) - except PlaywrightTimeoutError: - pass - try: - js_data = page.evaluate(self._JS_EXTRACT) or {} - except Exception: - pass - - if js_data.get("ok"): - vehicle_summary = self._build_car_from_js(js_data, url) - has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) - if not has_identity: - raise SiteStructureChangedError( - f"JS extraction returned no vehicle identity for {url}" - ) - db_record = self.car_mapper.map_to_car_record( - vehicle_url=url, - vehicle_summary=vehicle_summary, - payload_insights=self._build_payload_insights(vehicle_summary), - ) - else: - try: - page.wait_for_selector( - "#VehicleDetailViewModel, .veh-details, .vehicle-details, " - "[data-uname='vehicleDetailPage']", - timeout=400, - ) - except PlaywrightTimeoutError: - pass - page_html = page.content() - try: - dom_text = page.evaluate("() => document.body?.textContent || ''") - except Exception: - dom_text = "" - network_dump = { - "requests": [], - "json_responses": [], - "capture_limits": {}, - } - parsed = self.vehicle_parser.normalize(url, page_html, dom_text, network_dump) - self._raise_if_blocked_or_incomplete(parsed, url) - db_record = self.car_mapper.map_to_car_record( - vehicle_url=url, - vehicle_summary=parsed.get("vehicle_summary", {}), - payload_insights=parsed.get("payload_insights", {}), - ) - - record = CarRecord.model_validate(db_record.model_dump(mode="json")) - local_records.append(record) - logger.debug( - "[%d/%d] Parsed %s %s %s (fallback)", - global_idx, total, record.brand, record.model, record.year or "?", - ) - except Exception as exc: - if self._is_protection_or_network_error(exc): - local_protection += 1 - local_failed += 1 - local_failures.append({"vehicle_url": url, "error": str(exc)}) - logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) - finally: - if page is not None: - try: - page.close() - except Exception: - pass - return { - "records": local_records, - "failures": local_failures, - "cars_failed": local_failed, - "protection_events": local_protection, - } - - # Один page обрабатываем в главном потоке. - if n_pages == 1: - res = _process_slice(slices[0] if slices else []) - records.extend(res["records"]) - failures.extend(res["failures"]) - cars_failed += res["cars_failed"] - protection_events += res["protection_events"] - else: - # Несколько страниц — параллельно через потоки (каждый поток со своей Page). - with ThreadPoolExecutor(max_workers=n_pages) as executor: - futures = [executor.submit(_process_slice, s) for s in slices if s] - for fut in as_completed(futures): - res = fut.result() - records.extend(res["records"]) - failures.extend(res["failures"]) - cars_failed += res["cars_failed"] - protection_events += res["protection_events"] - - return { - "records": records, - "failures": failures, - "cars_failed": cars_failed, - "protection_events": protection_events, - } - - def sync_batch( - self, - vehicle_urls: list[str], - lane: str = "iaai_cars", - parallel_tabs: int | None = None, - ) -> dict: - # Пакетный скрапинг списка URL. - trace_id = self._new_trace_id("sync-batch") - started_at = time.perf_counter() - num_workers = parallel_tabs or self.settings.parallel_tabs - num_workers = min(num_workers, len(vehicle_urls), 48) - - cars_upserted = 0 - cars_failed = 0 - images_upserted = 0 - records: list[CarRecord] = [] - failures: list[dict[str, str]] = [] - http_successes = 0 - http_fallbacks = 0 - protection_events = 0 - - total = len(vehicle_urls) - logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) - - # Подготовка для HTTP fast-path: cookies + user-agent из Playwright сессии. - cookie_header = self._cookie_header_for_context() - user_agent = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " - "Gecko/20100101 Firefox/128.0" - ) - - # ── Phase 1: HTTP fast-path — все URLs одним ThreadPoolExecutor ── - fallback_urls: list[tuple[str, int]] = [] - - def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: - idx, url = idx_url - last_exc: Exception | None = None - for attempt in range(1 + self._HTTP_MAX_RETRIES): - try: - return idx, self._scrape_via_raw_http( - url, - cookie_header=cookie_header, - user_agent=user_agent, - ) - except Exception as exc: - last_exc = exc - if attempt < self._HTTP_MAX_RETRIES: - time.sleep(self._HTTP_RETRY_DELAYS[min(attempt, len(self._HTTP_RETRY_DELAYS) - 1)]) - return idx, last_exc # type: ignore[return-value] - - # Запускаем все URLs сразу — один пул потоков для минимального времени ожидания. - indexed_urls = list(enumerate(vehicle_urls)) - with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor: - for idx, result in executor.map(_fetch_one_with_retry, indexed_urls): - if isinstance(result, Exception): - http_fallbacks += 1 - logger.debug( - "[%d/%d] HTTP fast-path failed for %s: %s", - idx + 1, total, vehicle_urls[idx], result, - ) - fallback_urls.append((vehicle_urls[idx], idx + 1)) - else: - records.append(result) - http_successes += 1 - logger.debug( - "[%d/%d] Parsed %s %s %s (raw HTTP)", - idx + 1, total, result.brand, result.model, result.year or "?", - ) - - logger.info( - "HTTP phase done: %d OK, %d fallback (%.1fs)", - http_successes, http_fallbacks, time.perf_counter() - started_at, - ) - - # ── Phase 2: browser fallback (single-threaded — Playwright sync API is not thread-safe) ── - if fallback_urls: - logger.info( - "Fallback browser mode: %d/%d URLs, single page", - len(fallback_urls), total, - ) - fb_results = self._browser_fallback_parallel(fallback_urls, total, 1) - records.extend(fb_results["records"]) - cars_failed += fb_results["cars_failed"] - protection_events += fb_results["protection_events"] - failures.extend(fb_results["failures"]) - - # ── Phase 3: single DB flush ── - if records: - # Дедупликация перед записью. - seen_origins: set[str] = set() - unique_records: list[CarRecord] = [] - for rec in records: - key = rec.origin_id or rec.origin_url - if key not in seen_origins: - seen_origins.add(key) - unique_records.append(rec) - if len(unique_records) < len(records): - logger.info("Dedup before DB: %d → %d", len(records), len(unique_records)) - records = unique_records - - try: - batch_result = self.persistence.upsert_cars_batch(records) - cars_upserted = batch_result["inserted"] + batch_result["updated"] - images_upserted = batch_result["images_upserted"] - except Exception as exc: - logger.error("Batch upsert failed: %s", exc) - cars_failed += len(records) - - status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - return { - "trace_id": trace_id, - "status": status, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "http_successes": http_successes, - "http_fallbacks": http_fallbacks, - "protection_events": protection_events, - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "failures": failures, - } - - def sync_listing( - self, - make: str | None = None, - model: str | None = None, - lane: str = "iaai_cars", - limit: int | None = None, - only_new: bool | None = None, - ): - # Листинг + sync всех найденных машин. - # Применяем runtime_config как дефолты (CLI/API аргументы имеют приоритет). - rc = self.runtime_config.sync - if limit is None and rc.limit is not None: - limit = rc.limit - if only_new is None and rc.only_new is not None: - only_new = rc.only_new - if lane == "iaai_cars" and rc.lane is not None: - lane = rc.lane - - trace_id = self._new_trace_id("sync-listing") - started_at = time.perf_counter() - self.persistence.create_tables() - run_id = self.persistence.start_sync_run(lane=lane) - cars_upserted = 0 - cars_failed = 0 - cars_filtered = 0 - images_upserted = 0 - total = 0 - skipped_existing = 0 - failures: list[dict[str, str]] = [] - listing: dict = {} - - try: - effective_only_new = self.settings.sync_only_new if only_new is None else only_new - - # ── Сбор листинга ── - # При only_new + limit используем итеративный подход: - # листаем страницы одну за другой, фильтруем known на лету, - # останавливаемся когда набрали limit новых. - original_listing_cap = self.settings.listing.max_vehicles_per_run - original_include_pagination = self.settings.listing.include_pagination - original_collect_current_page_only = self.settings.listing.collect_current_page_only - original_max_pages_per_run = self.settings.listing.max_pages_per_run - - if effective_only_new and limit is not None and limit > 0: - # Итеративный сбор: страница→фильтр→проверка→следующая страница. - vehicle_urls, raw_urls, listing, skipped_existing = self._collect_listing_iterative( - make=make, model=model, limit=limit, - ) - else: - # Обычный сбор (без only_new или без limit). - prefetch_cap: int | None = None - if limit is not None and limit > 0: - prefetch_cap = limit - if prefetch_cap < original_listing_cap: - self.settings.listing.max_vehicles_per_run = prefetch_cap - - # Если включён режим только новых — заранее загружаем все известные origin_id, - # чтобы listing_collector мог остановиться при встрече старых страниц. - known_origin_ids: set[str] | None = None - if effective_only_new and self.settings.listing.early_stop_threshold > 0.0: - try: - known_origin_ids = self.persistence.get_all_origin_ids_for_lane("iaai:") - logger.info( - "Loaded %d known origin_ids for early-stop listing", - len(known_origin_ids), - ) - except Exception as exc: - logger.warning("Could not load known origin_ids for early-stop: %s", exc) - - try: - listing = self.collect_listing( - make=make, - model=model, - known_origin_ids=known_origin_ids, - ) - finally: - self.settings.listing.max_vehicles_per_run = original_listing_cap - self.settings.listing.include_pagination = original_include_pagination - self.settings.listing.collect_current_page_only = original_collect_current_page_only - self.settings.listing.max_pages_per_run = original_max_pages_per_run - - raw_urls = list(listing.get("vehicle_urls", [])) - vehicle_urls = self._dedupe_urls(raw_urls) - - if effective_only_new: - vehicle_urls, skipped_existing = self._filter_known_urls(vehicle_urls) - - if limit is not None: - vehicle_urls = vehicle_urls[:max(0, limit)] - - total = len(vehicle_urls) - logger.info("Starting sync: %d vehicles to process (batch mode)", total) - - # Собираем нормализованные origin_url для последующей пометки проданных. - # Используем URL (а не origin_id из URL), т.к. в БД origin_id берётся из parsed lot_number, - # который может отличаться от числа в URL листинга. - all_listing_origin_urls = set() - for raw_url in raw_urls: - normalized_url = self._normalize_vehicle_url(raw_url) - if normalized_url: - all_listing_origin_urls.add(normalized_url) - - # Обрабатываем пакетами. - batch_size = self.settings.celery.batch_size - for batch_start in range(0, total, batch_size): - batch_urls = vehicle_urls[batch_start:batch_start + batch_size] - logger.info( - "Processing batch %d-%d of %d", - batch_start + 1, min(batch_start + batch_size, total), total, - ) - batch_result = self.sync_batch(batch_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - - # Помечаем авто как проданные, если они исчезли из листинга. - # Только если сканирование было полным (не ограниченным limit/only_new/early_stop). - is_partial_scan = ( - effective_only_new - or (limit is not None and limit > 0) - or listing.get("early_stopped", False) - ) - if all_listing_origin_urls and not is_partial_scan: - try: - sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) - if sold_count: - logger.info("Marked %d cars as sold", sold_count) - except Exception as exc: - logger.warning("Failed to mark sold cars: %s", exc) - elif is_partial_scan: - logger.debug("Skipping mark_sold: partial/incremental scan (only_new=%s, limit=%s, early_stopped=%s)", - effective_only_new, limit, listing.get("early_stopped", False)) - except Exception as exc: - if not failures: - failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) - logger.error("sync_listing failed: %s", exc) - finally: - status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - error_summary = "; ".join(item["error"] for item in failures[:10]) if failures else None - self.persistence.finish_sync_run( - run_id, - status=status, - ids_fetched=total, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - images_upserted=images_upserted, - error_summary=error_summary, - ) - - logger.info( - "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images", - run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted, - ) - return { - "trace_id": trace_id, - "status": status, - "run_id": run_id, - "listing": listing, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "cars_filtered": cars_filtered, - "images_upserted": images_upserted, - "skipped_existing": skipped_existing, - "elapsed_seconds": round(time.perf_counter() - started_at, 3), - "failures": failures, - } - - def run_scheduled(self) -> None: - # Резерв для standalone-режима. - # В текущей архитектуре планирование выполняется через Celery beat + worker/tasks.py. - def _handle_shutdown(signum, frame): - logger.info("Received signal %s, shutting down gracefully...", signum) - self._shutdown_requested = True - - signal.signal(signal.SIGINT, _handle_shutdown) - signal.signal(signal.SIGTERM, _handle_shutdown) - - interval = self.settings.scheduler_interval_minutes * 60 - logger.info( - "Scheduler started: syncing every %d minutes", - self.settings.scheduler_interval_minutes, - ) - cycle = 0 - while not self._shutdown_requested: - cycle += 1 - logger.info("Scheduler cycle #%d starting", cycle) - start = time.time() - try: - if self.context: - try: - self.context.close() - except PlaywrightError: - pass - self.context = None - - if self.browser is None or self.playwright is None: - logger.info("Browser/Playwright not available, re-initializing...") - self.close() - self.__enter__() - - result = self.sync_listing() - elapsed = time.time() - start - logger.info( - "Cycle #%d done in %.1fs: %d upserted, %d failed", - cycle, elapsed, - result.get("cars_upserted", 0), - result.get("cars_failed", 0), - ) - except Exception as exc: - elapsed = time.time() - start - logger.error("Cycle #%d failed after %.1fs: %s", cycle, elapsed, exc) - try: - self.close() - except Exception: - pass - try: - self.__enter__() - except Exception as reinit_exc: - logger.error("Failed to re-initialize browser: %s", reinit_exc) - - if self._shutdown_requested: - break - - sleep_time = max(0, interval - (time.time() - start)) - if sleep_time > 0: - logger.info("Sleeping %.0f seconds until next cycle...", sleep_time) - slept = 0.0 - while slept < sleep_time and not self._shutdown_requested: - chunk = min(5.0, sleep_time - slept) - time.sleep(chunk) - slept += chunk - - logger.info("Scheduler stopped gracefully after %d cycles.", cycle) diff --git a/iaai_scraper/storage/__init__.py b/iaai_scraper/storage/__init__.py deleted file mode 100644 index c9c2ef6..0000000 --- a/iaai_scraper/storage/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__: list[str] = [] diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py deleted file mode 100644 index 22e6fda..0000000 --- a/iaai_scraper/storage/db.py +++ /dev/null @@ -1,576 +0,0 @@ -import logging -from contextlib import contextmanager -from datetime import datetime, timezone -from typing import Any, Iterator - -from sqlalchemy import create_engine, delete, or_, select, text, update -from sqlalchemy.dialects.postgresql import insert as pg_insert -from sqlalchemy.orm import Session, sessionmaker - -from ..core.config import Settings -from .models import Base, Car, Image, SyncRun -from .schemas import CarRecord - -logger = logging.getLogger("iaai_scraper.db") - - -CAR_DB_FIELDS = { - col.key for col in Car.__table__.columns - if col.key not in ("id",) -} - -_IN_CHUNK_SIZE = 5000 - - -class PersistenceService: - - def __init__(self, settings: Settings) -> None: - self.settings = settings - engine_kwargs = { - "echo": settings.database.echo, - "future": True, - } - if "postgresql" in settings.database.url: - engine_kwargs["pool_size"] = settings.database.pool_size - engine_kwargs["max_overflow"] = settings.database.max_overflow - engine_kwargs["pool_pre_ping"] = True - engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds - self.engine = create_engine(settings.database.url, **engine_kwargs) - self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) - - def create_tables(self) -> None: - # В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции. - is_sqlite = self.settings.database.url.startswith("sqlite") - if not is_sqlite and not self.settings.database.auto_create_tables: - return - try: - Base.metadata.create_all(self.engine) - except Exception: - logger.debug("create_tables skipped (schema already exists)") - - @contextmanager - def session_scope(self) -> Iterator[Session]: - session = self.session_factory() - try: - yield session - session.commit() - except Exception: - session.rollback() - raise - finally: - session.close() - - def start_sync_run(self, lane: str) -> int: - with self.session_scope() as session: - now = datetime.now(timezone.utc) - stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all() - for stale in stale_runs: - stale.status = "failed" - stale.finished_at = now - if not stale.error_summary: - stale.error_summary = "Recovered stale running sync run before starting a new run" - - run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) - session.add(run) - session.flush() - return int(run.id) - - def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: - with self.session_scope() as session: - run = session.get(SyncRun, run_id) - if run is None: - return - run.finished_at = datetime.now(timezone.utc) - run.status = status - run.ids_fetched = ids_fetched - run.cars_upserted = cars_upserted - run.cars_failed = cars_failed - run.images_upserted = images_upserted - run.error_summary = error_summary - - def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]: - if not origin_urls: - return set() - with self.session_scope() as session: - rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all() - return {str(row[0]) for row in rows if row and row[0]} - - def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]: - if not origin_ids: - return set() - with self.session_scope() as session: - rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all() - return {str(row[0]) for row in rows if row and row[0]} - - def get_existing_urls_and_ids( - self, origin_urls: list[str], origin_ids: list[str], - ) -> tuple[set[str], set[str]]: - # Загрузка существующих URL и origin_id. - if not origin_urls and not origin_ids: - return set(), set() - urls: set[str] = set() - ids: set[str] = set() - with self.session_scope() as session: - max_len = max(len(origin_urls), len(origin_ids), 1) - for i in range(0, max_len, _IN_CHUNK_SIZE): - url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] - id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] - conditions = [] - if url_chunk: - conditions.append(Car.origin_url.in_(url_chunk)) - if id_chunk: - conditions.append(Car.origin_id.in_(id_chunk)) - if not conditions: - continue - rows = session.execute( - select(Car.origin_url, Car.origin_id).where(or_(*conditions)) - ).all() - for r in rows: - if r[0]: - urls.add(str(r[0])) - if r[1]: - ids.add(str(r[1])) - return urls, ids - - @staticmethod - def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: - if not images: - return - session.add_all([ - Image( - fullres_image=str(img["fullres_image"]), - preview_image=str(img["preview_image"]), - order_index=int(img.get("order_index", 0)), - car_id=car_id, - ) - for img in images - ]) - - @staticmethod - def _car_payload(record: CarRecord) -> dict[str, object]: - payload = record.model_dump(mode="python") - return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} - - def _is_postgres(self) -> bool: - return self.engine.dialect.name == "postgresql" - - def _load_existing_cars( - self, - session: Session, - origin_ids: list[str], - origin_urls: list[str], - ) -> tuple[dict[str, Car], dict[str, Car]]: - existing_by_id: dict[str, Car] = {} - existing_by_url: dict[str, Car] = {} - if not origin_ids and not origin_urls: - return existing_by_id, existing_by_url - - existing_cars: list[Car] = [] - max_len = max(len(origin_ids), len(origin_urls), 1) - for i in range(0, max_len, _IN_CHUNK_SIZE): - id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] - url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] - conditions = [] - if id_chunk: - conditions.append(Car.origin_id.in_(id_chunk)) - if url_chunk: - conditions.append(Car.origin_url.in_(url_chunk)) - if not conditions: - continue - rows = session.execute( - select(Car).where(or_(*conditions)) - ).scalars().all() - existing_cars.extend(rows) - - for car in existing_cars: - if car.origin_id: - existing_by_id[car.origin_id] = car - if car.origin_url: - existing_by_url[car.origin_url] = car - return existing_by_id, existing_by_url - - def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]: - existing_images_map: dict[int, set[str]] = {} - if not car_ids: - return existing_images_map - - car_id_list = list(car_ids) - for i in range(0, len(car_id_list), _IN_CHUNK_SIZE): - chunk = car_id_list[i:i + _IN_CHUNK_SIZE] - img_rows = session.execute( - select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk)) - ).all() - for cid, furl in img_rows: - existing_images_map.setdefault(int(cid), set()).add(str(furl)) - return existing_images_map - - @staticmethod - def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]: - return { - key: getattr(insert_stmt.excluded, key) - for key in CAR_DB_FIELDS - } - - def _replace_images_for_car( - self, - session: Session, - car_id: int, - images: list[dict[str, object]], - origin_id: str, - ) -> int: - nested = session.begin_nested() - try: - session.execute(delete(Image).where(Image.car_id == car_id)) - self._add_images(session, car_id, images) - session.flush() - nested.commit() - return len(images) - except Exception: - nested.rollback() - logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True) - return 0 - - def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]: - inserted = 0 - updated = 0 - images_total = 0 - - with self.session_scope() as session: - origin_ids = [r.origin_id for r in records if r.origin_id] - origin_urls = [r.origin_url for r in records if r.origin_url] - existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) - - entries: list[dict[str, object]] = [] - upsert_payloads: list[dict[str, object]] = [] - for record in records: - payload = self._car_payload(record) - images = [image.model_dump(mode="python") for image in record.images] - car_by_id = existing_by_id.get(record.origin_id) - car_by_url = existing_by_url.get(record.origin_url) - entry: dict[str, object] = {"record": record, "images": images, "car_id": None} - - if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None: - for key, value in payload.items(): - setattr(car_by_url, key, value) - car_by_url.last_seen_at = record.last_seen_at - entry["car_id"] = int(car_by_url.id) - updated += 1 - else: - upsert_payloads.append(payload) - if car_by_id is not None: - updated += 1 - else: - inserted += 1 - entries.append(entry) - - session.flush() - - if upsert_payloads: - insert_stmt = pg_insert(Car).values(upsert_payloads) - upsert_stmt = insert_stmt.on_conflict_do_update( - index_elements=[Car.origin_id], - set_=self._postgres_upsert_set_map(insert_stmt), - ).returning(Car.id, Car.origin_id) - rows = session.execute(upsert_stmt).all() - car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows} - for entry in entries: - if entry["car_id"] is not None: - continue - record = entry["record"] - car_id = car_ids_by_origin_id.get(record.origin_id) - if car_id is None: - raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}") - entry["car_id"] = car_id - - car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None} - existing_images_map = self._load_existing_image_urls(session, car_ids) - images_by_car_id: dict[int, list[dict[str, object]]] = {} - replace_ids: list[int] = [] - - for entry in entries: - car_id = int(entry["car_id"]) - images = entry["images"] - new_image_urls = { - str(img.get("fullres_image", "")) - for img in images - if img.get("fullres_image") - } - old_image_urls = existing_images_map.get(car_id, set()) - if new_image_urls != old_image_urls: - replace_ids.append(car_id) - images_by_car_id[car_id] = images - else: - images_total += len(old_image_urls) - - if replace_ids: - for i in range(0, len(replace_ids), _IN_CHUNK_SIZE): - chunk = replace_ids[i:i + _IN_CHUNK_SIZE] - session.execute(delete(Image).where(Image.car_id.in_(chunk))) - for car_id in replace_ids: - images = images_by_car_id[car_id] - self._add_images(session, car_id, images) - images_total += len(images) - - return {"inserted": inserted, "updated": updated, "images_upserted": images_total} - - def upsert_car(self, record: CarRecord): - # Вставка или обновление автомобиля по origin_id/origin_url. - payload = self._car_payload(record) - images = [image.model_dump(mode="python") for image in record.images] - with self.session_scope() as session: - if self._is_postgres(): - car_by_url = session.execute( - select(Car).where(Car.origin_url == record.origin_url) - ).scalar_one_or_none() - if car_by_url is not None and car_by_url.origin_id != record.origin_id: - for key, value in payload.items(): - setattr(car_by_url, key, value) - car_by_url.last_seen_at = record.last_seen_at - session.flush() - car_id = int(car_by_url.id) - action = "updated" - else: - existed = session.execute( - select(Car.id).where(Car.origin_id == record.origin_id) - ).scalar_one_or_none() is not None - insert_stmt = pg_insert(Car).values(**payload) - upsert_stmt = insert_stmt.on_conflict_do_update( - index_elements=[Car.origin_id], - set_=self._postgres_upsert_set_map(insert_stmt), - ).returning(Car.id) - car_id = int(session.execute(upsert_stmt).scalar_one()) - action = "updated" if existed or car_by_url is not None else "inserted" - - images_upserted = self._replace_images_for_car( - session, car_id, images, record.origin_id, - ) - return {"car_id": car_id, "images_upserted": images_upserted, "action": action} - - car = session.execute( - select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) - ).scalar_one_or_none() - action = "inserted" - if car is None: - car = Car(**payload) - session.add(car) - session.flush() - else: - action = "updated" - for key, value in payload.items(): - setattr(car, key, value) - car.last_seen_at = record.last_seen_at - session.flush() - images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id) - return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action} - self._add_images(session, int(car.id), images) - session.flush() - return {"car_id": int(car.id), "images_upserted": len(images), "action": action} - def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]: - """Пакетный upsert нескольких автомобилей в одной транзакции. - - Оптимизации: - - Дедупликация записей по origin_id перед вставкой. - - Chunked IN-queries для больших списков (обход лимита PG параметров). - - Пропуск перезаписи изображений, если набор URL не изменился. - - Один DELETE по car_id IN (...) вместо удаления по одному. - - Fallback на по-одному upsert если batch commit упал. - """ - # ── Дедупликация записей внутри батча ── - seen_ids: dict[str, int] = {} - unique_records: list[CarRecord] = [] - for idx, r in enumerate(records): - key = r.origin_id or r.origin_url - if key in seen_ids: - logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key]) - continue - seen_ids[key] = idx - unique_records.append(r) - - if len(unique_records) < len(records): - logger.info("Deduped batch: %d → %d records", len(records), len(unique_records)) - records = unique_records - - try: - return self._upsert_cars_batch_inner(records) - except Exception as exc: - logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc) - return self._upsert_cars_individually(records) - - def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]: - """Внутренняя реализация batched upsert (одна транзакция).""" - if self._is_postgres(): - return self._upsert_cars_batch_postgres(records) - - inserted = 0 - updated = 0 - images_total = 0 - - with self.session_scope() as session: - # Получаем существующие записи chunked-запросами. - origin_ids = [r.origin_id for r in records if r.origin_id] - origin_urls = [r.origin_url for r in records if r.origin_url] - - existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) - - # Предзагружаем ВСЕ изображения для обновляемых машин одним запросом. - existing_car_ids = set() - for record in records: - car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) - if car is not None: - existing_car_ids.add(int(car.id)) - - # Строим маппинг car_id → set(image_urls) для сравнения. - existing_images_map = self._load_existing_image_urls(session, existing_car_ids) - - new_cars: list[tuple[Car, list[dict]]] = [] - update_cars_needing_images: list[tuple[Car, list[dict]]] = [] - - for record in records: - payload = self._car_payload(record) - images = [image.model_dump(mode="python") for image in record.images] - - car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) - if car is None: - car = Car(**payload) - session.add(car) - inserted += 1 - new_cars.append((car, images)) - else: - for key, value in payload.items(): - setattr(car, key, value) - car.last_seen_at = record.last_seen_at - updated += 1 - - # Проверяем, изменились ли изображения. - new_image_urls = {img.get("fullres_image", "") for img in images} - old_image_urls = existing_images_map.get(int(car.id), set()) - if new_image_urls != old_image_urls: - update_cars_needing_images.append((car, images)) - else: - images_total += len(old_image_urls) - - # Один flush для всех вставок. - session.flush() - - # Добавляем изображения для новых автомобилей. - for car, images in new_cars: - self._add_images(session, int(car.id), images) - images_total += len(images) - - # Массово обновляем изображения только для машин с изменёнными картинками. - if update_cars_needing_images: - update_ids = [int(car.id) for car, _ in update_cars_needing_images] - for i in range(0, len(update_ids), _IN_CHUNK_SIZE): - chunk = update_ids[i:i + _IN_CHUNK_SIZE] - session.execute(delete(Image).where(Image.car_id.in_(chunk))) - for car, images in update_cars_needing_images: - self._add_images(session, int(car.id), images) - images_total += len(images) - - return {"inserted": inserted, "updated": updated, "images_upserted": images_total} - - def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]: - # Fallback на поштучный upsert. - inserted = 0 - updated = 0 - images_total = 0 - for record in records: - try: - result = self.upsert_car(record) - action = result.get("action", "inserted") - if action == "inserted": - inserted += 1 - else: - updated += 1 - images_total += int(result.get("images_upserted", 0)) - except Exception as exc: - logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) - return {"inserted": inserted, "updated": updated, "images_upserted": images_total} - - def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int: - """Помечает авто как проданные, если их нет в активном листинге (по origin_id).""" - if not active_origin_ids: - return 0 - with self.session_scope() as session: - stmt = ( - update(Car) - .where(Car.origin_id.notin_(active_origin_ids)) - .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like("iaai:%")) - .values(is_sold=True) - ) - result = session.execute(stmt) - count = result.rowcount or 0 - if count: - logger.info("Marked %d cars as sold (no longer in listing)", count) - return count - - def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int: - """Помечает авто как проданные, если их URL нет в активном листинге. - - Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, - что кардинально быстрее при больших объёмах (100K+ URLs). - """ - if not active_origin_urls: - return 0 - - is_postgres = "postgresql" in self.settings.database.url - - with self.session_scope() as session: - if is_postgres: - # Создаём временную таблицу с активными URL. - session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP")) - session.execute(text("TRUNCATE _active_urls")) - - # Вставляем активные URL чанками. - url_list = list(active_origin_urls) - for i in range(0, len(url_list), _IN_CHUNK_SIZE): - chunk = url_list[i:i + _IN_CHUNK_SIZE] - values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) - params = {f"u{j}": url for j, url in enumerate(chunk)} - session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params) - - # Создаём индекс на временной таблице для ускорения JOIN. - session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)")) - - # Массовая пометка проданных в PostgreSQL. - result = session.execute(text(""" - UPDATE cars - SET is_sold = TRUE - FROM ( - SELECT c.id - FROM cars c - LEFT JOIN _active_urls a ON c.origin_url = a.url - WHERE a.url IS NULL - AND c.is_sold = FALSE - AND c.origin_id LIKE 'iaai:%%' - ) sub - WHERE cars.id = sub.id - """)) - count = result.rowcount or 0 - else: - # Упрощённый путь для SQLite. - stmt = ( - update(Car) - .where(Car.origin_url.notin_(active_origin_urls)) - .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like("iaai:%")) - .values(is_sold=True) - ) - result = session.execute(stmt) - count = result.rowcount or 0 - - if count: - logger.info("Marked %d cars as sold by URL (no longer in listing)", count) - return count - - def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]: - """Возвращает все известные origin_id для заданного префикса. - - Использует yield_per для потоковой загрузки при большом количестве записей. - """ - with self.session_scope() as session: - result = session.execute( - select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000) - ) - return {str(row[0]) for row in result if row and row[0]} \ No newline at end of file diff --git a/iaai_scraper/storage/enums.py b/iaai_scraper/storage/enums.py deleted file mode 100644 index d25d8a5..0000000 --- a/iaai_scraper/storage/enums.py +++ /dev/null @@ -1,24 +0,0 @@ -CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") -DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA") -GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") -STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA") -BODY_TYPE_ENUM_VALUES = ( - "COUPE", - "SUV", - "HATCHBACK", - "MINIVAN", - "SEDAN", - "STATION_WAGON", - "PICKUP", - "TRUCK", - "OPEN", - "RV", - "OTHER", - "NA", -) -COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") -ORIGIN_ENUM_VALUES = ( - "IAAI", - "NA", -) -SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA") diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py deleted file mode 100644 index 29d5566..0000000 --- a/iaai_scraper/storage/models.py +++ /dev/null @@ -1,82 +0,0 @@ -from datetime import datetime - -from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship - -from .enums import ( - BODY_TYPE_ENUM_VALUES, - COUNTRY_ENUM_VALUES, - CURRENCY_ENUM_VALUES, - DRIVE_ENUM_VALUES, - GEARBOX_ENUM_VALUES, - ORIGIN_ENUM_VALUES, - SELLING_TYPE_ENUM_VALUES, - STEERING_WHEEL_ENUM_VALUES, -) - - -class Base(DeclarativeBase): - pass - - -class Car(Base): - __tablename__ = "cars" - __table_args__ = ( - Index("ix_cars_brand_model", "brand", "model"), - Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"), - ) - id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) - parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) - brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True) - model: Mapped[str] = mapped_column(String(50), nullable=False) - year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) - price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) - currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD") - mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA") - is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) - color: Mapped[str] = mapped_column(String(), nullable=False, default="other") - drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True) - gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True) - steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True) - body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER") - engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) - selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA") - one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA") - origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) - origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) - is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) - non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) - rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - slug: Mapped[str] = mapped_column(String(), nullable=False) - last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) - images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") - - -class Image(Base): - __tablename__ = "images" - id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) - fullres_image: Mapped[str] = mapped_column(String(), nullable=False) - preview_image: Mapped[str] = mapped_column(String(), nullable=False) - order_index: Mapped[int] = mapped_column(Integer, nullable=False) - car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True) - car: Mapped[Car] = relationship("Car", back_populates="images") - - -class SyncRun(Base): - __tablename__ = "sync_runs" - id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) - started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) - finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - status: Mapped[str] = mapped_column(Text, nullable=False, index=True) - lane: Mapped[str] = mapped_column(Text, nullable=False) - ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/iaai_scraper/storage/schemas.py b/iaai_scraper/storage/schemas.py deleted file mode 100644 index 7af7c43..0000000 --- a/iaai_scraper/storage/schemas.py +++ /dev/null @@ -1,87 +0,0 @@ -from datetime import datetime, timezone -from pydantic import BaseModel, ConfigDict, Field - - -class ImageRecord(BaseModel): - fullres_image: str - preview_image: str - order_index: int = 0 - - -class CarRecord(BaseModel): - parser_id: str - brand: str - model: str - year: int | None = None - price: int | None = None - currency: str = "USD" - mileage: int = 0 - country: str = "US" - is_sold: bool = False - color: str = "other" - drive: str | None = None - gearbox: str | None = None - steering_wheel: str | None = None - body_type: str = "OTHER" - engine_volume: int | None = None - selling_type: str = "AUCTION" - one_owner: bool = False - new_car: bool = False - is_hidden: bool = False - origin: str = "NA" - origin_url: str - origin_id: str - is_damaged: bool = False - evaluation: str | None = None - non_smoking: bool = True - rental: bool = False - repair_history: bool = False - slug: str - last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) - images: list[ImageRecord] = Field(default_factory=list) - - -class ImageRead(BaseModel): - model_config = ConfigDict(from_attributes=True) - - id: int - fullres_image: str - preview_image: str - order_index: int = 0 - - -class CarRead(BaseModel): - model_config = ConfigDict(from_attributes=True) - - id: int - parser_id: str - brand: str - model: str - year: int | None = None - price: int | None = None - currency: str = "USD" - mileage: int = 0 - country: str = "US" - is_sold: bool = False - color: str = "other" - drive: str | None = None - gearbox: str | None = None - steering_wheel: str | None = None - body_type: str = "OTHER" - engine_volume: int | None = None - selling_type: str = "AUCTION" - one_owner: bool = False - new_car: bool = False - is_hidden: bool = False - origin: str = "NA" - origin_url: str = "" - origin_id: str = "" - is_damaged: bool = False - evaluation: str | None = None - non_smoking: bool = True - rental: bool = False - repair_history: bool = False - slug: str = "" - last_seen_at: datetime | None = None - images: list[ImageRead] = Field(default_factory=list) - diff --git a/iaai_scraper/worker/__init__.py b/iaai_scraper/worker/__init__.py deleted file mode 100644 index 841be13..0000000 --- a/iaai_scraper/worker/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .celery_app import celery_app - -__all__ = ["celery_app"] diff --git a/iaai_scraper/worker/celery_app.py b/iaai_scraper/worker/celery_app.py deleted file mode 100644 index 4f5682b..0000000 --- a/iaai_scraper/worker/celery_app.py +++ /dev/null @@ -1,56 +0,0 @@ -# Инициализация Celery-приложения и периодических задач. - -from celery import Celery - -from ..core.config import settings - - -def _broker_url() -> str: - return settings.celery.broker_url or settings.redis.url - - -def _result_backend() -> str: - return settings.celery.result_backend or settings.redis.url - - -celery_app = Celery( - "iaai_scraper", - broker=_broker_url(), - backend=_result_backend(), -) - -celery_app.conf.update( - task_serializer="json", - accept_content=["json"], - result_serializer="json", - timezone="UTC", - enable_utc=True, - task_soft_time_limit=settings.celery.task_soft_time_limit, - task_time_limit=settings.celery.task_time_limit, - task_acks_late=True, - task_reject_on_worker_lost=True, - task_track_started=True, - worker_concurrency=settings.celery.worker_concurrency, - worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child, - worker_pool="prefork", - worker_prefetch_multiplier=1, - broker_connection_retry_on_startup=True, - broker_transport_options={ - "visibility_timeout": settings.celery.broker_visibility_timeout, - }, - result_expires=86400, - beat_schedule={ - "periodic-sync-listing": { - "task": "iaai_scraper.worker.tasks.sync_listing_task", - "schedule": settings.celery.beat_sync_interval_minutes * 60.0, - "args": (), - "kwargs": {"limit": settings.celery.beat_sync_limit}, - "options": {"queue": "scraping"}, - } - }, - task_routes={ - "iaai_scraper.worker.tasks.*": {"queue": "scraping"} - }, -) - -celery_app.autodiscover_tasks(["iaai_scraper.worker"]) \ No newline at end of file diff --git a/iaai_scraper/worker/tasks.py b/iaai_scraper/worker/tasks.py deleted file mode 100644 index e244f3a..0000000 --- a/iaai_scraper/worker/tasks.py +++ /dev/null @@ -1,221 +0,0 @@ -# Задачи Celery для синхронизации автомобилей и листинга IAAI. - -from concurrent.futures import ThreadPoolExecutor -import logging -from threading import Event, Thread -import time -import uuid - -from celery import shared_task -from redis import Redis - -from ..core.config import Settings -from ..scraper import IAAIScraper -from ..storage.db import PersistenceService - -logger = logging.getLogger("iaai_scraper.worker.tasks") - -SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing" - - -def _run_browser_job(func, *args, **kwargs): - # Браузерный код запускаем в отдельном потоке без активного loop. - with ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser") as executor: - future = executor.submit(func, *args, **kwargs) - return future.result() - - -def _sync_listing_lock_ttl_seconds() -> int: - settings = Settings() - # Небольшой запас к лимиту времени, чтобы lock снимался после сбоев. - return max(settings.celery.task_time_limit + 120, 300) - - -def _get_persistence() -> PersistenceService: - return PersistenceService(Settings()) - - -def _get_redis() -> Redis: - settings = Settings() - return Redis.from_url(settings.redis.url, decode_responses=True) - - -def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: - try: - return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) - except Exception as exc: - logger.warning("Failed to acquire lock %s", key, exc_info=True) - return False - - -def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: - try: - refreshed = redis_client.eval( - """ - if redis.call('GET', KEYS[1]) == ARGV[1] then - return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) - end - return 0 - """, - 1, - key, - owner_token, - int(ttl_seconds), - ) - return bool(refreshed) - except Exception as exc: - logger.warning("Failed to refresh lock %s", key, exc_info=True) - return None - - -def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: - try: - redis_client.eval( - """ - if redis.call('GET', KEYS[1]) == ARGV[1] then - return redis.call('DEL', KEYS[1]) - end - return 0 - """, - 1, - key, - owner_token, - ) - except Exception as exc: - logger.warning("Failed to release lock %s", key, exc_info=True) - - -def _start_lock_heartbeat( - redis_client: Redis, - key: str, - owner_token: str, - ttl_seconds: int, -) -> tuple[Event, Thread]: - stop_event = Event() - interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) - - def _heartbeat() -> None: - while not stop_event.wait(interval_seconds): - refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) - if refreshed is False: - logger.warning("Lost sync_listing lock ownership for %s", owner_token) - return - - thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) - thread.start() - return stop_event, thread - - -@shared_task( - name="iaai_scraper.worker.tasks.sync_vehicle_task", - bind=True, - max_retries=2, - default_retry_delay=30, - acks_late=True, -) -def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"): - # Скрапинг и upsert одного автомобиля. - persistence = _get_persistence() - persistence.create_tables() - - try: - def _job(): - with IAAIScraper() as scraper: - return scraper.sync_vehicle(vehicle_url, lane=lane) - - result = _run_browser_job(_job) - logger.info("sync_vehicle_task completed: %s", vehicle_url) - return { - "status": "success", - "vehicle_url": vehicle_url, - "db_action": result.get("db_action"), - "images_upserted": result.get("images_upserted", 0), - } - - except Exception as exc: - logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True) - raise self.retry(exc=exc) - - -@shared_task( - name="iaai_scraper.worker.tasks.sync_listing_task", - bind=True, - max_retries=1, - default_retry_delay=120, - acks_late=True, -) -def sync_listing_task( - self, - make: str | None = None, - model: str | None = None, - lane: str = "iaai_cars", - limit: int | None = None, - only_new: bool | None = None, -): - # Полный цикл: листинг + sync всех найденных машин. - persistence = _get_persistence() - persistence.create_tables() - task_id = self.request.id or "unknown" - owner_token = f"{task_id}:{uuid.uuid4().hex}" - redis_client = _get_redis() - - lock_acquired = False - lock_ttl = _sync_listing_lock_ttl_seconds() - heartbeat_stop: Event | None = None - heartbeat_thread: Thread | None = None - - lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) - - if not lock_acquired: - logger.info("sync_listing_task skipped: another sync is already running") - return { - "status": "skipped", - "reason": "sync_already_running", - "task_id": task_id, - } - - try: - heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( - redis_client, - SYNC_LISTING_LOCK_KEY, - owner_token, - lock_ttl, - ) - self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) - def _job(): - with IAAIScraper() as scraper: - return scraper.sync_listing( - make=make, - model=model, - lane=lane, - limit=limit, - only_new=only_new, - ) - - result = _run_browser_job(_job) - - summary = { - "task_id": task_id, - "run_id": result.get("run_id"), - "cars_upserted": result.get("cars_upserted", 0), - "cars_failed": result.get("cars_failed", 0), - "images_upserted": result.get("images_upserted", 0), - "skipped_existing": result.get("skipped_existing", 0), - "elapsed_seconds": result.get("elapsed_seconds"), - } - logger.info( - "sync_listing_task completed: %d upserted, %d failed", - summary["cars_upserted"], summary["cars_failed"], - ) - return {"status": "success", **summary} - - except Exception as exc: - logger.error("sync_listing_task failed: %s", exc, exc_info=True) - raise self.retry(exc=exc) - finally: - if heartbeat_stop is not None: - heartbeat_stop.set() - if heartbeat_thread is not None: - heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) - if lock_acquired: - _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) diff --git a/tests/test_listing.py b/tests/test_listing.py deleted file mode 100644 index aae31aa..0000000 --- a/tests/test_listing.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -import unittest - -from iaai_scraper.browser.pace import HumanPacer -from iaai_scraper.core.config import Settings -from iaai_scraper.browser.listing import ListingCollector - - -class _FakePage: - def __init__(self, counts: dict[str, int]) -> None: - self._counts = counts - - class _Locator: - def __init__(self, count_value: int) -> None: - self._count_value = count_value - - def count(self) -> int: - return self._count_value - - def locator(self, selector: str) -> "_FakePage._Locator": - return _FakePage._Locator(self._counts.get(selector, 0)) - - -class TestListingUnit(unittest.TestCase): - def test_has_next_page_true_for_known_selector(self) -> None: - page = _FakePage({"a[aria-label*='Next']": 1}) - self.assertTrue(ListingCollector._has_next_page(page)) - - def test_has_next_page_false_when_no_selectors(self) -> None: - page = _FakePage({}) - self.assertFalse(ListingCollector._has_next_page(page)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_mappers.py b/tests/test_mappers.py deleted file mode 100644 index 4ac34be..0000000 --- a/tests/test_mappers.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import annotations - -import unittest - -from iaai_scraper.parsing.mapper import CarMapper - - -class TestCarMapper(unittest.TestCase): - def setUp(self) -> None: - self.mapper = CarMapper() - - def test_deduplicates_images_by_image_key(self) -> None: - urls = [ - "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", - "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", - ] - - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/123~US", - vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, - payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, - ) - - self.assertEqual(len(record.images), 2) - self.assertIn("width=845", record.images[0].fullres_image) - self.assertIn("height=633", record.images[0].fullres_image) - - def test_no_damage_marker_is_not_damaged(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/123~US", - vehicle_summary={"make": "Ford", "model": "Focus"}, - payload_insights={ - "vehicle_core": {}, - "pricing": {}, - "damage": {"primary": "normal wear"}, - "auction": {}, - "images": {}, - }, - ) - - self.assertFalse(record.is_damaged) - - def test_unknown_and_empty_values_fallbacks(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/999~US", - vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, - payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, - ) - - self.assertEqual(record.brand, "UNKNOWN") - self.assertEqual(record.model, "UNKNOWN") - self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None) - self.assertEqual(record.drive, "NA") - self.assertEqual(record.gearbox, "NA") - - def test_mapper_handles_case_and_spaces(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/888~US", - vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, - payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, - ) - - self.assertEqual(record.drive, "FWD") - self.assertEqual(record.gearbox, "AT") - - def test_price_parsing_dirty_formats(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/777~US", - vehicle_summary={"make": "Toyota", "model": "Corolla"}, - payload_insights={ - "vehicle_core": {}, - "pricing": {"buy_now": "USD 4,500 - 5,200"}, - "damage": {}, - "auction": {}, - "images": {}, - }, - ) - - self.assertEqual(record.price, 5200) - - def test_currency_detection_from_symbol(self) -> None: - record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/778~US", - vehicle_summary={"make": "Toyota", "model": "Corolla"}, - payload_insights={ - "vehicle_core": {}, - "pricing": {"buy_now": "€4.500,00"}, - "damage": {}, - "auction": {}, - "images": {}, - }, - ) - - self.assertEqual(record.currency, "EUR") - self.assertEqual(record.price, 4500) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py deleted file mode 100644 index 9884e17..0000000 --- a/tests/test_parser.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import unittest - -from iaai_scraper.parsing.parser import VehicleParser - - -class TestVehicleParserUnit(unittest.TestCase): - def setUp(self) -> None: - self.parser = VehicleParser() - - def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None: - dom_text = """ - Stock #: - 45089484 - Primary Damage: - Front End - Odometer: - 50,123 mi (Actual) - """ - result = self.parser._parse_dom_key_value_pairs(dom_text) - self.assertEqual(result.get("lot_number"), "45089484") - self.assertEqual(result.get("primary_damage"), "Front End") - self.assertEqual(result.get("odometer"), "50,123 mi (Actual)") - - def test_parse_title_for_year_make_model(self) -> None: - parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") - self.assertEqual(parsed["year"], "2014") - self.assertEqual(parsed["make"], "TOYOTA") - self.assertEqual(parsed["model"], "CAMRY") - - def test_extract_image_urls_filters_other_vehicle(self) -> None: - vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" - payloads = [ - { - "imageUrls": [ - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", - ] - } - ] - urls = self.parser._extract_image_urls(payloads, "", vehicle_url) - self.assertEqual(len(urls), 1) - self.assertIn("45089484", urls[0]) - - def test_extract_image_urls_deduplicates_same_url(self) -> None: - vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" - payloads = [{"imageUrls": [ - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - ]}] - urls = self.parser._extract_image_urls(payloads, "", vehicle_url) - self.assertEqual(len(urls), 1) - - def test_dom_hints_detect_captcha_and_antibot(self) -> None: - hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") - self.assertTrue(hints["has_captcha_text"]) - self.assertTrue(hints["has_antibot_text"]) - - def test_access_notes_reflect_antibot_signals(self) -> None: - summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."} - notes = self.parser._build_access_notes(summary, []) - self.assertTrue(notes["possible_captcha"]) - self.assertTrue(notes["possible_antibot"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py deleted file mode 100644 index c053fbf..0000000 --- a/tests/test_scraper.py +++ /dev/null @@ -1,174 +0,0 @@ -from __future__ import annotations - -import unittest -from unittest.mock import MagicMock - -from iaai_scraper.core.config import Settings -from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError -from iaai_scraper.scraper import IAAIScraper -from iaai_scraper.storage.schemas import CarRecord - - -def make_db_record(origin_id: str) -> dict[str, object]: - return CarRecord( - parser_id=f"iaai:{origin_id}", - brand="Toyota", - model="Camry", - origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", - origin_id=origin_id, - slug=f"toyota-camry-{origin_id}", - ).model_dump(mode="json") - - -class TestScraperSync(unittest.TestCase): - def _make_scraper(self) -> IAAIScraper: - s = Settings() - s.log_level = "CRITICAL" - s.database.url = "sqlite://" - return IAAIScraper(s) - - def test_sync_vehicle_uses_db_record_without_remapping(self) -> None: - scraper = self._make_scraper() - - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=1) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) - - scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) - scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) - - result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") - - self.assertEqual(result["status"], "success") - self.assertIn("trace_id", result) - self.assertIn("elapsed_seconds", result) - self.assertEqual(scraper.persistence.upsert_car.call_count, 1) - - def test_sync_listing_uses_db_record_without_remapping(self) -> None: - scraper = self._make_scraper() - - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=2) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set())) - - scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]}) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [], - }) - - result = scraper.sync_listing() - - self.assertEqual(result["cars_upserted"], 1) - self.assertEqual(result["cars_failed"], 0) - self.assertIn("trace_id", result) - self.assertIn("elapsed_seconds", result) - scraper.sync_batch.assert_called_once() - - def test_sync_listing_respects_limit(self) -> None: - scraper = self._make_scraper() - - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=3) - scraper.persistence.finish_sync_run = MagicMock() - - # Проверка пути only_new с limit. - scraper._collect_listing_iterative = MagicMock(return_value=( - ["https://www.iaai.com/VehicleDetail/222~US"], - ["https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/VehicleDetail/333~US"], - {"vehicle_urls": [], "pages_collected": 1, "early_stopped": False}, - 0, - )) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [], - }) - - scraper.sync_listing(limit=1) - - # Должен уйти только один URL. - scraper.sync_batch.assert_called_once() - batch_urls = scraper.sync_batch.call_args[0][0] - self.assertEqual(len(batch_urls), 1) - - def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None: - scraper = self._make_scraper() - - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=5) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( - {"https://www.iaai.com/VehicleDetail/111~US"}, - {"iaai:222"}, - )) - - scraper.collect_listing = MagicMock(return_value={ - "vehicle_urls": [ - "https://www.iaai.com/VehicleDetail/111~US", # exists by URL - "https://www.iaai.com/VehicleDetail/222~US", # exists by ID - "https://www.iaai.com/VehicleDetail/333~US", # new - ] - }) - # Возвращаем результат для одного нового авто. - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], - }) - - result = scraper.sync_listing(only_new=True) - - self.assertEqual(result["skipped_existing"], 2) - self.assertEqual(result["cars_upserted"], 1) - # В batch должен попасть только новый URL. - scraper.sync_batch.assert_called_once() - batch_urls = scraper.sync_batch.call_args[0][0] - self.assertEqual(len(batch_urls), 1) - self.assertIn("333", batch_urls[0]) - scraper.persistence.get_existing_urls_and_ids.assert_called_once() - - def test_close_resets_browser_state(self) -> None: - scraper = self._make_scraper() - http_pool = MagicMock() - scraper._http_pool = http_pool - scraper.context = MagicMock() - scraper.browser = MagicMock() - scraper.playwright = MagicMock() - - scraper.close() - - http_pool.clear.assert_called_once() - self.assertIsNone(scraper._http_pool) - self.assertIsNone(scraper.context) - self.assertIsNone(scraper.browser) - self.assertIsNone(scraper.playwright) - - def test_guard_raises_on_antibot_signals(self) -> None: - with self.assertRaises(AntiBotDetectedError): - IAAIScraper._raise_if_blocked_or_incomplete( - { - "dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, - "access_notes": {}, - "vehicle_summary": {}, - }, - "https://www.iaai.com/VehicleDetail/999~US", - ) - - def test_guard_raises_on_empty_vehicle_page(self) -> None: - with self.assertRaises(SiteStructureChangedError): - IAAIScraper._raise_if_blocked_or_incomplete( - { - "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, - "access_notes": {"possible_captcha": False, "possible_antibot": False}, - "vehicle_summary": {}, - }, - "https://www.iaai.com/VehicleDetail/999~US", - ) - - def test_is_protection_or_network_error_detects_known_signals(self) -> None: - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) - self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) - - -if __name__ == "__main__": - unittest.main()