From a4c93a1df116e26a7eea8ae69bc5f02765d6e9a4 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Mon, 13 Apr 2026 16:35:08 +0300 Subject: [PATCH] fix docker scraping improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup --- .env.example | 46 +- Dockerfile | 7 +- README.md | 8 +- alembic/versions/001_initial.py | 13 +- alembic/versions/002_add_indexes.py | 20 +- alembic/versions/003_add_composite_indexes.py | 38 + docker-compose.yml | 28 +- iaai_scraper/api/deps.py | 2 +- iaai_scraper/api/routes/health.py | 5 +- iaai_scraper/browser/factory.py | 151 +- iaai_scraper/browser/listing.py | 167 ++- iaai_scraper/browser/network.py | 10 +- iaai_scraper/browser/pace.py | 2 +- iaai_scraper/core/config.py | 81 +- iaai_scraper/core/exceptions.py | 6 +- iaai_scraper/core/logs.py | 2 +- iaai_scraper/core/utils.py | 37 +- iaai_scraper/parsing/mapper.py | 45 +- iaai_scraper/parsing/parser.py | 120 +- iaai_scraper/scraper.py | 1229 +++++++++++++---- iaai_scraper/storage/db.py | 466 ++++++- iaai_scraper/storage/enums.py | 9 - iaai_scraper/storage/models.py | 17 +- iaai_scraper/worker/tasks.py | 150 +- pyproject.toml | 1 + tests/test_db.py | 10 - tests/test_listing.py | 6 - tests/test_parser.py | 6 +- tests/test_scraper.py | 89 +- tests/test_worker_tasks.py | 92 ++ 30 files changed, 2239 insertions(+), 624 deletions(-) create mode 100644 alembic/versions/003_add_composite_indexes.py create mode 100644 tests/test_worker_tasks.py diff --git a/.env.example b/.env.example index 12f9f89..9fa711f 100644 --- a/.env.example +++ b/.env.example @@ -10,25 +10,25 @@ IAAI_MAX_CAPTURED_JSON_RESPONSES=30 # Sequential listing-first mode. IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars IAAI_MAX_PAGES_PER_RUN=10 -IAAI_MAX_VEHICLES_PER_RUN=30 -IAAI_PAGE_LINK_LIMIT=100 +IAAI_MAX_VEHICLES_PER_RUN=50000 +IAAI_PAGE_LINK_LIMIT=500 IAAI_INCLUDE_PAGINATION=true IAAI_COLLECT_CURRENT_PAGE_ONLY=false -# Human-like pacing. -IAAI_HUMAN_PACE_ENABLED=true -IAAI_AFTER_LISTING_OPEN_MIN_S=2.5 -IAAI_AFTER_LISTING_OPEN_MAX_S=4.5 -IAAI_AFTER_FILTER_ACTION_MIN_S=2.0 -IAAI_AFTER_FILTER_ACTION_MAX_S=4.0 -IAAI_BEFORE_VEHICLE_OPEN_MIN_S=1.5 -IAAI_BEFORE_VEHICLE_OPEN_MAX_S=3.0 -IAAI_AFTER_VEHICLE_OPEN_MIN_S=1.0 -IAAI_AFTER_VEHICLE_OPEN_MAX_S=2.5 -IAAI_BETWEEN_VEHICLES_MIN_S=3.0 -IAAI_BETWEEN_VEHICLES_MAX_S=6.0 -IAAI_AFTER_PAGE_CHANGE_MIN_S=3.0 -IAAI_AFTER_PAGE_CHANGE_MAX_S=6.0 +# Паузы (отключены для максимальной скорости; включи на VPS если попадаешь под блокировки). +IAAI_HUMAN_PACE_ENABLED=false +IAAI_AFTER_LISTING_OPEN_MIN_S=0.2 +IAAI_AFTER_LISTING_OPEN_MAX_S=0.5 +IAAI_AFTER_FILTER_ACTION_MIN_S=0.2 +IAAI_AFTER_FILTER_ACTION_MAX_S=0.5 +IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02 +IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08 +IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02 +IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08 +IAAI_BETWEEN_VEHICLES_MIN_S=0.02 +IAAI_BETWEEN_VEHICLES_MAX_S=0.08 +IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2 +IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5 # Sync settings IAAI_SYNC_ONLY_NEW=true @@ -40,16 +40,6 @@ IAAI_RETRY_DELAY_SECONDS=2.5 IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 IAAI_RETRY_JITTER_SECONDS=0.25 -# Session persistence (cookies + localStorage) -# Keeps browser session alive between runs (up to 30 days). -IAAI_STORAGE_STATE_PATH=storage_state.json -IAAI_SESSION_MAX_AGE_DAYS=30 -IAAI_SESSION_SAVE_ON_EXIT=true - -# IAAI login credentials (for authenticated scraping) -# IAAI_LOGIN_EMAIL=your@email.com -# IAAI_LOGIN_PASSWORD=your_password - # Proxy (HTTP/HTTPS preferred for Playwright) # Chromium does not support SOCKS5 proxy authentication directly. # Use residential or mobile USA proxy. @@ -60,8 +50,8 @@ IAAI_SESSION_SAVE_ON_EXIT=true # Database (PostgreSQL). IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper IAAI_DATABASE_ECHO=false -IAAI_DATABASE_POOL_SIZE=5 -IAAI_DATABASE_MAX_OVERFLOW=10 +IAAI_DATABASE_POOL_SIZE=10 +IAAI_DATABASE_MAX_OVERFLOW=20 # ─── Redis (Celery broker) ───────────────────────────────── IAAI_REDIS_URL=redis://redis:6379/0 diff --git a/Dockerfile b/Dockerfile index cd4e2a0..45b69b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,14 @@ WORKDIR /app COPY pyproject.toml uv.lock ./ RUN pip install --no-cache-dir uv \ && uv export --format requirements-txt --no-dev --no-hashes --no-emit-project --frozen -o /tmp/requirements.txt \ - && pip install --no-cache-dir -r /tmp/requirements.txt + && pip install --no-cache-dir -r /tmp/requirements.txt \ + && pip install --no-cache-dir playwright-stealth + +# Install Firefox browser (headless-friendly, bypasses Incapsula) +RUN python -m playwright install firefox COPY . . +RUN pip install --no-cache-dir -e . RUN python -m compileall -q iaai_scraper RUN chmod +x entrypoint.sh RUN chown -R app:app /app diff --git a/README.md b/README.md index 88e144c..14a25db 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,12 @@ Docker **не нужен**. Данные хранятся в SQLite-файле ` git clone cd iaai_scraper_project pip install -e . -playwright install chromium +playwright install firefox iaai init-db ``` +`iaai init-db` актуален для SQLite/локального CLI-режима. Для PostgreSQL используйте Alembic-миграции (`alembic upgrade head` или сервис `migrate` в Docker). + Готовый `.env` уже в репозитории и настроен на SQLite ничего менять не нужно. ### Запуск парсера @@ -157,6 +159,8 @@ iaai sync-vehicle "https://www.iaai.com/VehicleDetail/45089484~US" iaai sync-listing --limit 10 ``` +`iaai init-db` используйте для SQLite/локальных тестов. В PostgreSQL-сценарии применяйте миграции Alembic. + ## Структура ```text @@ -267,8 +271,6 @@ uv run pytest -q - `only_new` - `limit` -Так же используются: `lane`, `only_new`, `limit`. - ### Секция `filters` Поддерживаются поля: diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py index 4c31205..4179d3e 100644 --- a/alembic/versions/001_initial.py +++ b/alembic/versions/001_initial.py @@ -1,9 +1,4 @@ -"""Initial schema — cars, images, sync_runs - -Revision ID: 001_initial -Revises: -Create Date: 2026-04-08 -""" +# Начальная схема: cars, images, sync_runs from typing import Sequence, Union from alembic import op @@ -16,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # --- cars --- + # Таблица cars — аукционные машины с IAAI op.create_table( "cars", sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), @@ -53,7 +48,7 @@ def upgrade() -> None: op.create_index("ix_cars_origin_url", "cars", ["origin_url"]) op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True) - # --- images --- + # Таблица images — изображения машин op.create_table( "images", sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), @@ -63,7 +58,7 @@ def upgrade() -> None: sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False), ) - # --- sync_runs --- + # Таблица sync_runs — логирование синхронизаций op.create_table( "sync_runs", sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), diff --git a/alembic/versions/002_add_indexes.py b/alembic/versions/002_add_indexes.py index 284f3d8..2c7b631 100644 --- a/alembic/versions/002_add_indexes.py +++ b/alembic/versions/002_add_indexes.py @@ -1,9 +1,4 @@ -"""Add performance indexes for growing database - -Revision ID: 002_add_indexes -Revises: 001_initial -Create Date: 2026-04-09 -""" +# Индексы производительности from typing import Sequence, Union from alembic import op @@ -15,25 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # cars: ускорение фильтрации по бренду в API и статистике op.create_index("ix_cars_brand", "cars", ["brand"]) - - # cars: составной индекс бренд+модель для комбинированных фильтров op.create_index("ix_cars_brand_model", "cars", ["brand", "model"]) - - # cars: ускорение фильтрации по году (year_min/year_max) op.create_index("ix_cars_year", "cars", ["year"]) - - # cars: ускорение фильтрации по статусу продажи op.create_index("ix_cars_is_sold", "cars", ["is_sold"]) - - # cars: ускорение сортировки ORDER BY last_seen_at DESC (пагинация) op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"]) - - # images: ускорение JOIN/DELETE по car_id (критично при upsert) op.create_index("ix_images_car_id", "images", ["car_id"]) - - # sync_runs: ускорение поиска stale runs по статусу op.create_index("ix_sync_runs_status", "sync_runs", ["status"]) diff --git a/alembic/versions/003_add_composite_indexes.py b/alembic/versions/003_add_composite_indexes.py new file mode 100644 index 0000000..1388d45 --- /dev/null +++ b/alembic/versions/003_add_composite_indexes.py @@ -0,0 +1,38 @@ +# Индексы для масштабированной БД (20k+ записей) +from typing import Sequence, Union + +from alembic import op + +revision: str = "003_add_composite_indexes" +down_revision: Union[str, None] = "002_add_indexes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Partial index для активных машин IAAI (is_sold = FALSE) + # Ускоряет поиск при mark_sold операциях + op.execute( + "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai " + "ON cars (origin_url) " + "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'" + ) + + # Composite index для проверки дубликатов изображений + op.create_index( + "ix_images_car_id_fullres", + "images", + ["car_id", "fullres_image"], + ) + + # Index для быстрого поиска existing машин по origin_url + op.execute( + "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id " + "ON cars (origin_url, origin_id)" + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id") + op.drop_index("ix_images_car_id_fullres", table_name="images") + op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai") diff --git a/docker-compose.yml b/docker-compose.yml index d3a2362..93cc784 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,26 @@ x-app-env: &app-env - IAAI_DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper} + # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here + IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800} - CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} - CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} + CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300} + CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600} CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200} - CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-20} + # Без лимита beat может забирать слишком большой объём за один запуск. + # Консервативный дефолт для anti-fraud: 300 авто за запуск. + CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-300} + CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} + CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} + IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40} + IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true} + IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999} + IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000} + IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true} IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json} IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} - IAAI_STORAGE_STATE_PATH: ${IAAI_STORAGE_STATE_PATH:-/data/storage_state.json} - IAAI_SESSION_MAX_AGE_DAYS: ${IAAI_SESSION_MAX_AGE_DAYS:-30} - IAAI_SESSION_SAVE_ON_EXIT: ${IAAI_SESSION_SAVE_ON_EXIT:-true} - IAAI_LOGIN_EMAIL: ${IAAI_LOGIN_EMAIL:-} - IAAI_LOGIN_PASSWORD: ${IAAI_LOGIN_PASSWORD:-} + IAAI_BROWSER_ENGINE: ${IAAI_BROWSER_ENGINE:-auto} TZ: ${TZ:-UTC} x-env-file: &env-file @@ -135,9 +141,9 @@ services: stop_grace_period: 60s command: > celery -A iaai_scraper.worker.celery_app worker - --loglevel=info --concurrency=1 --pool=prefork + --loglevel=info --concurrency=4 --pool=prefork --pidfile=/tmp/celery-worker.pid - -Q scraping --max-tasks-per-child=20 + -Q scraping --max-tasks-per-child=5 healthcheck: test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"] interval: 60s diff --git a/iaai_scraper/api/deps.py b/iaai_scraper/api/deps.py index 60d59e4..86b4bc3 100644 --- a/iaai_scraper/api/deps.py +++ b/iaai_scraper/api/deps.py @@ -1,4 +1,4 @@ -# Dependency helpers для FastAPI-роутов. +# Вспомогательные зависимости для FastAPI-роутов. from fastapi import Request diff --git a/iaai_scraper/api/routes/health.py b/iaai_scraper/api/routes/health.py index 99ac070..a7de8d7 100644 --- a/iaai_scraper/api/routes/health.py +++ b/iaai_scraper/api/routes/health.py @@ -1,5 +1,7 @@ # Роут проверки доступности сервиса и соединения с БД. +import logging + from fastapi import APIRouter, Depends from sqlalchemy import text @@ -7,6 +9,7 @@ from ..deps import get_persistence from ...storage.db import PersistenceService router = APIRouter() +logger = logging.getLogger("iaai_scraper.api.health") @router.get("/health") @@ -18,7 +21,7 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)): session.execute(text("SELECT 1")) db_ok = True except Exception: - pass + logger.warning("Health DB check failed", exc_info=True) return { "status": "ok" if db_ok else "degraded", diff --git a/iaai_scraper/browser/factory.py b/iaai_scraper/browser/factory.py index 91cc726..69b393c 100644 --- a/iaai_scraper/browser/factory.py +++ b/iaai_scraper/browser/factory.py @@ -4,13 +4,18 @@ import random from playwright.sync_api import Browser, BrowserContext, Playwright +try: + from playwright_stealth import stealth_sync +except ImportError: + stealth_sync = None + from ..core.config import Settings logger = logging.getLogger("iaai_scraper.browser") def _build_init_script() -> str: - # JS-патч признаков автоматизации. + # Патч признаков автоматизации. hardware_concurrency = random.choice([4, 8, 12, 16]) device_memory = random.choice([4, 8, 16]) languages = ["en-US", "en"] @@ -63,17 +68,61 @@ class BrowserFactory: def __init__(self, settings: Settings) -> None: self.settings = settings + def _resolve_engine(self) -> str: + # Выбор движка браузера. + engine = self.settings.browser_engine.strip().lower() + if engine == "auto": + return "firefox" if self.settings.headless else "chromium" + if engine in ("firefox", "chromium"): + return engine + logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine) + return "firefox" if self.settings.headless else "chromium" + def create_browser(self, playwright: Playwright) -> Browser: - launch_kwargs: dict = { - "headless": self.settings.headless, - "args": [ - "--disable-blink-features=AutomationControlled", - "--no-default-browser-check", - "--disable-dev-shm-usage", - "--disable-features=IsolateOrigins,site-per-process", - ], - } + engine = self._resolve_engine() proxy_dict = self.settings.proxy.to_playwright_dict() + + if engine == "firefox": + launch_kwargs: dict = {"headless": self.settings.headless} + if proxy_dict: + launch_kwargs["proxy"] = proxy_dict + logger.info("Using proxy: %s", self.settings.proxy.server) + # Параметры Firefox для headless-режима. + launch_kwargs["firefox_user_prefs"] = { + "dom.webdriver.enabled": False, + "useAutomationExtension": False, + # Базовые оптимизации для VPS. + "media.autoplay.default": 5, + "media.volume_scale": "0.0", + "media.audio.playback.standalone": False, + "dom.ipc.processCount": 1, + "dom.ipc.plugins.enabled": False, + "browser.cache.disk.enable": False, + "browser.cache.memory.enable": True, + "browser.cache.memory.max_entry_size": 8192, + "network.prefetch-next": False, + "network.dns.disablePrefetch": True, + "permissions.default.image": 2, + "javascript.options.mem.gc_incremental_mark_slice_ms": 20, + } + logger.info("Launching Firefox (headless=%s)", self.settings.headless) + return playwright.firefox.launch(**launch_kwargs) + + # Путь запуска Chromium. + args = [ + "--disable-blink-features=AutomationControlled", + "--no-default-browser-check", + "--disable-dev-shm-usage", + "--disable-features=IsolateOrigins,site-per-process", + ] + if self.settings.headless: + args.append("--headless=new") + pw_headless = False + logger.info("Using Chromium new-headless mode (--headless=new)") + else: + pw_headless = False + + launch_kwargs = {"headless": pw_headless, "args": args} if proxy_dict: launch_kwargs["proxy"] = proxy_dict logger.info("Using proxy: %s", self.settings.proxy.server) @@ -84,34 +133,80 @@ class BrowserFactory: logger.warning("Chrome channel launch failed, falling back to Chromium") return playwright.chromium.launch(**launch_kwargs) - def create_context(self, browser: Browser, storage_state: str | None = None) -> BrowserContext: + def create_context(self, browser: Browser) -> BrowserContext: viewport = random.choice(self.settings.fingerprint.viewport_presets) timezone_id = random.choice(self.settings.fingerprint.timezone_candidates) color_scheme = random.choice(["light", "dark"]) - context = browser.new_context( - storage_state=storage_state or None, - user_agent=self.settings.fingerprint.user_agent, - viewport=viewport, - screen=viewport, - device_scale_factor=random.choice([1, 1.25]), - is_mobile=False, - has_touch=False, - locale=self.settings.fingerprint.locale, - timezone_id=timezone_id, - color_scheme=color_scheme, - java_script_enabled=True, - ignore_https_errors=False, - extra_http_headers={ + is_firefox = browser.browser_type.name == "firefox" + + ctx_kwargs: dict = { + "viewport": viewport, + "screen": viewport, + "locale": self.settings.fingerprint.locale, + "timezone_id": timezone_id, + "color_scheme": color_scheme, + "java_script_enabled": True, + "ignore_https_errors": False, + } + + if is_firefox: + # Заголовки и user-agent для Firefox. + ctx_kwargs["user_agent"] = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + ctx_kwargs["extra_http_headers"] = { + "Accept-Language": "en-US,en;q=0.5", + "DNT": "1", + "Upgrade-Insecure-Requests": "1", + } + else: + ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent + ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25]) + ctx_kwargs["is_mobile"] = False + ctx_kwargs["has_touch"] = False + ctx_kwargs["extra_http_headers"] = { "Accept-Language": "en-US,en;q=0.9", "DNT": "1", "Upgrade-Insecure-Requests": "1", "Sec-CH-UA": self.settings.fingerprint.sec_ch_ua, "Sec-CH-UA-Mobile": "?0", "Sec-CH-UA-Platform": '"Windows"', - }, - ) + } + + context = browser.new_context(**ctx_kwargs) context.set_default_timeout(self.settings.default_timeout_ms) context.set_default_navigation_timeout(self.settings.default_timeout_ms) - context.add_init_script(_build_init_script()) + + if not is_firefox: + # Патчи маскировки для Chromium. + context.add_init_script(_build_init_script()) + if stealth_sync: + context.on("page", lambda page: stealth_sync(page)) + logger.debug("playwright-stealth attached to context") + return context + + @staticmethod + def enable_resource_blocking(page) -> None: + # Блокируем тяжёлые ресурсы для ускорения загрузки страниц. + BLOCKED_TYPES = {"image", "stylesheet", "font", "media"} + BLOCKED_URL_PATTERNS = ( + "google-analytics", "googletagmanager", "facebook.net", + "doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2", + "analytics", "tracking", "adservice", + ) + + def _handle_route(route): + req = route.request + if req.resource_type in BLOCKED_TYPES: + route.abort() + return + url = req.url.lower() + if any(pat in url for pat in BLOCKED_URL_PATTERNS): + route.abort() + return + route.continue_() + + page.route("**/*", _handle_route) diff --git a/iaai_scraper/browser/listing.py b/iaai_scraper/browser/listing.py index 302081a..943bd84 100644 --- a/iaai_scraper/browser/listing.py +++ b/iaai_scraper/browser/listing.py @@ -1,5 +1,6 @@ import logging import re +import time from dataclasses import asdict, dataclass, field from typing import Any from urllib.parse import urljoin @@ -11,7 +12,7 @@ from ..core.config import Settings from ..core.utils import first_non_empty logger = logging.getLogger("iaai_scraper.listing") -VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/\d+(?:~[A-Z]{2})?", re.IGNORECASE) +VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) @dataclass(slots=True) @@ -37,15 +38,35 @@ class ListingCollector: def open_cars_listing(self, page: Page) -> None: logger.info("Opening cars listing page: %s", self.settings.listing.cars_url) - page.goto(self.settings.listing.cars_url, wait_until="domcontentloaded") + url = self.settings.listing.cars_url + last_err = None + for attempt in range(3): + try: + page.goto(url, wait_until="commit", timeout=60_000) + last_err = None + break + except Exception as e: + last_err = e + logger.warning("goto listing attempt %d failed: %s", attempt + 1, e) + # Небольшой backoff при ошибках открытия листинга. + time.sleep(5 * (attempt + 1)) + if last_err: + logger.warning("All goto attempts failed, trying JS navigation") + try: + page.evaluate(f"window.location.href = '{url}'") + except Exception: + pass + # Быстрая проверка готовности страницы. try: - page.wait_for_load_state("networkidle", timeout=15000) + page.wait_for_load_state("domcontentloaded", timeout=12_000) except Exception: - logger.debug("networkidle timeout on listing page, continuing with current state") + pass try: - page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=20000) + page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=4_000) except Exception: - logger.warning("Vehicle links did not appear within timeout; page may not have rendered fully") + # Короткая пауза вместо длинного sleep. + time.sleep(0.25) + logger.info("Listing page URL: %s", page.url) self.pacer.after_listing_open() def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]: @@ -59,28 +80,48 @@ class ListingCollector: return applied def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: - anchors = page.locator("a[href*='/VehicleDetail/']") - total = min(anchors.count(), self.settings.listing.page_link_limit) + # Считываем ссылки одним проходом по DOM. + raw_items = page.eval_on_selector_all( + "a[href*='/VehicleDetail/']", + """ + (nodes) => nodes.map((a) => ({ + href: a.getAttribute('href') || '', + title: a.getAttribute('title') || '', + text: (a.textContent || '').trim(), + })) + """, + ) + total = min(len(raw_items), self.settings.listing.page_link_limit) links: list[ListingVehicleLink] = [] seen: set[str] = set() for idx in range(total): - anchor = anchors.nth(idx) - href = anchor.get_attribute("href") or "" + item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} + href = str(item.get("href") or "") match = VEHICLE_HREF_RE.search(href) if not match: continue + lot_number = match.group(1) absolute = urljoin(self.settings.home_url, match.group(0)) if absolute in seen: continue seen.add(absolute) - title = first_non_empty([anchor.get_attribute("title"), anchor.text_content(), ""]) or "" - links.append(ListingVehicleLink(href=absolute, title=str(title).strip())) + title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" + links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) if len(links) >= self.settings.listing.max_vehicles_per_run: break next_page_detected = self._has_next_page(page) return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) def go_to_next_page(self, page: Page) -> bool: + # Запоминаем первую ссылку текущей страницы для определения смены контента. + old_first_href = "" + try: + first_link = page.locator("a[href*='/VehicleDetail/']").first + if first_link.count() > 0: + old_first_href = first_link.get_attribute("href") or "" + except Exception: + pass + selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"] for selector in selectors: locator = page.locator(selector).first @@ -93,32 +134,118 @@ class ListingCollector: continue self.pacer.move_mouse_to(page, locator) locator.click() + + # Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться. + if old_first_href: + try: + page.wait_for_function( + f"""() => {{ + const a = document.querySelector("a[href*='/VehicleDetail/']"); + return a && a.getAttribute('href') !== '{old_first_href}'; + }}""", + timeout=8000, + ) + except Exception: + pass + else: + try: + page.wait_for_load_state("domcontentloaded", timeout=15000) + except Exception: + pass + try: - page.wait_for_load_state("networkidle", timeout=15000) + page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000) except Exception: pass self.pacer.after_page_change() return True return False - def collect_listing_links(self, page: Page, *, make: str | None = None, model: str | None = None) -> dict[str, Any]: + def collect_listing_links( + self, + page: Page, + *, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + ) -> dict[str, Any]: self.open_cars_listing(page) applied_filters = self.apply_filters(page, make=make, model=model) pages: list[dict[str, object]] = [] all_links: list[str] = [] + early_stopped = False + threshold = self.settings.listing.early_stop_threshold + for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1): page_result = self.collect_current_page(page, page_number=page_number) - pages.append({"page_number": page_result.page_number, "source_url": page_result.source_url, "links_found": len(page_result.vehicle_links), "vehicle_links": [asdict(item) for item in page_result.vehicle_links], "next_page_detected": page_result.next_page_detected}) + pages.append({ + "page_number": page_result.page_number, + "source_url": page_result.source_url, + "links_found": len(page_result.vehicle_links), + "vehicle_links": [asdict(item) for item in page_result.vehicle_links], + "next_page_detected": page_result.next_page_detected, + }) for item in page_result.vehicle_links: if item.href not in all_links: all_links.append(item.href) if len(all_links) >= self.settings.listing.max_vehicles_per_run: break - if len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination or not page_result.next_page_detected: + + # Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла. + # Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице). + if ( + known_origin_ids is not None + and threshold > 0.0 + and page_result.vehicle_links + ): + page_known = sum( + 1 for item in page_result.vehicle_links + if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids + ) + page_new = len(page_result.vehicle_links) - page_known + ratio = page_known / len(page_result.vehicle_links) + # Останавливаемся только если нет новых И порог превышен + if ratio >= threshold and page_new == 0: + logger.info( + "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", + page_number, ratio * 100, page_known, + len(page_result.vehicle_links), threshold * 100, + ) + early_stopped = True + break + elif page_new > 0 and ratio >= threshold: + logger.info( + "Page %d: %.0f%% known but %d new found — продолжаем", + page_number, ratio * 100, page_new, + ) + + if ( + len(all_links) >= self.settings.listing.max_vehicles_per_run + or self.settings.listing.collect_current_page_only + or not self.settings.listing.include_pagination + or not page_result.next_page_detected + ): break if not self.go_to_next_page(page): break - return {"listing_url": self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages), "vehicles_collected": len(all_links), "vehicle_urls": all_links, "pages": pages, "strategy": {"sequential": True, "collect_current_page_only": self.settings.listing.collect_current_page_only, "include_pagination": self.settings.listing.include_pagination, "max_pages_per_run": self.settings.listing.max_pages_per_run, "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run}} + + return { + "listing_url": self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": len(pages), + "vehicles_collected": len(all_links), + "vehicle_urls": all_links, + "early_stopped": early_stopped, + "pages": pages, + "strategy": { + "sequential": True, + "collect_current_page_only": self.settings.listing.collect_current_page_only, + "include_pagination": self.settings.listing.include_pagination, + "max_pages_per_run": self.settings.listing.max_pages_per_run, + "max_vehicles_per_run": self.settings.listing.max_vehicles_per_run, + "early_stop_threshold": threshold, + }, + } @staticmethod def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool: @@ -131,7 +258,11 @@ class ListingCollector: locator.fill(value) page.keyboard.press("Enter") try: - page.wait_for_load_state("networkidle", timeout=15000) + page.wait_for_load_state("domcontentloaded", timeout=15000) + except Exception: + pass + try: + page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000) except Exception: pass return True diff --git a/iaai_scraper/browser/network.py b/iaai_scraper/browser/network.py index 11b403a..0674976 100644 --- a/iaai_scraper/browser/network.py +++ b/iaai_scraper/browser/network.py @@ -13,7 +13,7 @@ logger = logging.getLogger("iaai_scraper.network") @dataclass class NetworkCapture: - # XHR/fetch перехватчик. + # Перехватчик сетевых запросов. settings: Settings requests: list[dict[str, Any]] = field(default_factory=list) @@ -21,13 +21,21 @@ class NetworkCapture: _seen_req: set[str] = field(default_factory=set) _seen_resp: set[str] = field(default_factory=set) _origin: str | None = None + _page: Any = field(default=None) def attach(self, page: Page, origin_url: str | None = None) -> None: + # Снимаем старые подписки перед повторным attach. + try: + page.remove_listener("request", self._on_request) + page.remove_listener("response", self._on_response) + except Exception: + pass # Подписка на сетевые события try: self._origin = urlparse(origin_url or page.url).netloc.lower() or None except Exception: self._origin = None + self._page = page page.on("request", self._on_request) page.on("response", self._on_response) diff --git a/iaai_scraper/browser/pace.py b/iaai_scraper/browser/pace.py index f9e4042..10c2551 100644 --- a/iaai_scraper/browser/pace.py +++ b/iaai_scraper/browser/pace.py @@ -7,7 +7,7 @@ from ..core.config import Settings class HumanPacer: - # Random паузы между действиями. + # Случайные паузы между действиями. def __init__(self, settings: Settings) -> None: self.settings = settings diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py index 393768e..4690527 100644 --- a/iaai_scraper/core/config.py +++ b/iaai_scraper/core/config.py @@ -84,18 +84,18 @@ class CaptureConfig: @dataclass(slots=True) class HumanPaceConfig: enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True) - after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 2.5) - after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 4.5) - after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 2.0) - after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 4.0) - before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.5) - before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 1.5) - after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.3) - after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 1.0) - between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.5) - between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 1.5) - after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 3.0) - after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 6.0) + after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5) + after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2) + after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5) + after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2) + before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) + before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) + after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05) + after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15) + between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05) + between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15) + after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8) + after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8) # Конфиг сбора листинга (URL, лимиты страниц и машин) @@ -103,11 +103,14 @@ class HumanPaceConfig: @dataclass(slots=True) class ListingConfig: cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") - max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 5) - max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 100) - page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 200) + max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999) + max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000) + page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500) include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True) collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False) + # Порог раннего останова: если доля уже известных машин на странице >= этого значения, + # прекращаем листать — все новые машины уже найдены. 0 = отключено. + early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8) # - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) @@ -135,13 +138,16 @@ class RedisConfig: class CeleryConfig: broker_url: str = _env_str("CELERY_BROKER_URL", "") result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") - task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 600) - task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 900) - worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 1) - worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 20) + task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) + task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) + worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) + worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) - beat_sync_limit: int = _env_int("CELERY_BEAT_SYNC_LIMIT", 26) + beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None + batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) + parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8) + block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True) # --- Конфиг прокси (server, username, password) --- @@ -167,38 +173,22 @@ class ProxyConfig: return result -# --- Конфиг сессии браузера (куки, storage_state) --- - -@dataclass(slots=True) -class SessionConfig: - storage_state_path: str | None = _env_optional_str("IAAI_STORAGE_STATE_PATH") - max_age_days: int = _env_int("IAAI_SESSION_MAX_AGE_DAYS", 30) - save_on_exit: bool = _env_bool("IAAI_SESSION_SAVE_ON_EXIT", True) - login_email: str | None = _env_optional_str("IAAI_LOGIN_EMAIL") - login_password: str | None = _env_optional_str("IAAI_LOGIN_PASSWORD") - login_url: str = _env_str("IAAI_LOGIN_URL", "https://www.iaai.com/Login") - - @property - def enabled(self) -> bool: - return bool(self.storage_state_path) - - @property - def has_credentials(self) -> bool: - return bool(self.login_email and self.login_password) - - # --- Главный объект настроек: собирает все блоки конфигурации --- @dataclass(slots=True) class Settings: home_url: str = "https://www.iaai.com/" default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000) - network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 800) + network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400) + fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000) + fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1) + fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000) max_retries: int = _env_int("IAAI_MAX_RETRIES", 3) retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5) retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0) retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25) headless: bool = _env_bool("IAAI_HEADLESS", True) + browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto") log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO") log_file: str | None = _env_optional_str("IAAI_LOG_FILE") enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True) @@ -215,7 +205,14 @@ class Settings: redis: RedisConfig = field(default_factory=RedisConfig) celery: CeleryConfig = field(default_factory=CeleryConfig) proxy: ProxyConfig = field(default_factory=ProxyConfig) - session: SessionConfig = field(default_factory=SessionConfig) + + @property + def parallel_tabs(self) -> int: + return self.celery.parallel_tabs + + @property + def block_resources(self) -> bool: + return self.celery.block_resources # Глобальный синглтон — используется по умолчанию во всех модулях. settings = Settings() diff --git a/iaai_scraper/core/exceptions.py b/iaai_scraper/core/exceptions.py index dd4c3dc..f984981 100644 --- a/iaai_scraper/core/exceptions.py +++ b/iaai_scraper/core/exceptions.py @@ -1,10 +1,10 @@ class ScraperError(Exception): - """Base scraper exception.""" + """Базовое исключение скрапера.""" class AntiBotDetectedError(ScraperError): - """Raised when the target website appears to block automation.""" + """Вызывается, когда сайт блокирует автоматизацию.""" class SiteStructureChangedError(ScraperError): - """Raised when the page shape changed and required data is missing.""" + """Вызывается, когда структура страницы изменилась и данных не хватает.""" diff --git a/iaai_scraper/core/logs.py b/iaai_scraper/core/logs.py index 4e621d8..32a312d 100644 --- a/iaai_scraper/core/logs.py +++ b/iaai_scraper/core/logs.py @@ -2,7 +2,7 @@ import logging import sys from contextvars import ContextVar -# ContextVar хранит trace_id текущего потока/корутины. +# Храним trace_id текущего потока/корутины. TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") diff --git a/iaai_scraper/core/utils.py b/iaai_scraper/core/utils.py index 875d73c..8c8cad5 100644 --- a/iaai_scraper/core/utils.py +++ b/iaai_scraper/core/utils.py @@ -1,7 +1,7 @@ import json import re from pathlib import Path -from typing import Any, Iterable +from typing import Any, Callable, Iterable def save_to_json(data: Any, filename: str | Path) -> None: @@ -17,7 +17,7 @@ def first_non_empty(values: Iterable[Any]) -> Any | None: return None -# Regex для VIN, lot, price. +# Регулярные выражения для VIN, lot и price. VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) LOT_RE = re.compile(r"\b(\d{7,10})\b") PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") @@ -37,3 +37,36 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = for item in obj: found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) return found + + +def deep_find_all_keys( + payloads: list, + field_map: dict[str, set[str]], + max_depth: int = 64, +) -> dict[str, list]: + """Извлекает все нужные поля за один проход по JSON.""" + # Готовим обратную карту: нормализованный ключ -> имя поля. + reverse: dict[str, str] = {} + for field_name, keys in field_map.items(): + for k in keys: + reverse[k.lower()] = field_name + + result: dict[str, list] = {f: [] for f in field_map} + + def _recurse(obj: Any, depth: int) -> None: + if depth >= max_depth: + return + if isinstance(obj, dict): + for k, v in obj.items(): + field = reverse.get(k.lower()) + if field is not None: + result[field].append(v) + _recurse(v, depth + 1) + elif isinstance(obj, list): + for item in obj: + _recurse(item, depth + 1) + + for payload in payloads: + _recurse(payload, 0) + + return result diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py index cc119a0..2c1f79f 100644 --- a/iaai_scraper/parsing/mapper.py +++ b/iaai_scraper/parsing/mapper.py @@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord class CarMapper: - # IAAI data → CarRecord. + # Преобразование данных IAAI в CarRecord. BODY_MAP = { "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", @@ -72,12 +72,18 @@ class CarMapper: ], self._to_money_int, ) - mileage = self._first_parsed_int( - [core.get("odometer"), vehicle_summary.get("odometer"), 0], - self._to_int, - ) or 0 + mileage = self._parse_odometer( + first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")]) + ) color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) + # Пытаемся определить привод из строки двигателя. + if not drive or drive == "NA": + engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")])) + if engine_text: + inferred_drive = self._normalize_drive(engine_text) + if inferred_drive and inferred_drive != "NA": + drive = inferred_drive gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) @@ -107,7 +113,7 @@ class CarMapper: ] ) ) - slug = self._slugify(" ".join(filter(None, [str(year or ""), brand, model, origin_id]))) + slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))) images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) origin = "IAAI" @@ -206,6 +212,25 @@ class CarMapper: return parsed return None + @staticmethod + def _parse_odometer(value: Any) -> int: + # Разбор пробега из строк IAAI. + if value is None: + return 0 + text = str(value).strip() + if not text: + return 0 + lowered = text.lower() + if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]): + return 0 + # Извлекаем число из строкового формата одометра. + numbers = re.findall(r"[\d,]+", text) + for num_str in numbers: + clean = num_str.replace(",", "") + if clean.isdigit() and int(clean) > 0: + return int(clean) + return 0 + def _to_engine_cc(self, value: Any) -> int | None: # Поддержка литров и cc. text = str(value).lower().strip() if value is not None else "" @@ -344,13 +369,13 @@ class CarMapper: return preview return url - # ---------- parser_id ---------- + # Формирование parser_id. - _PARSER_ID_ALPHABET = ascii_letters + digits # a-zA-Z0-9 + _PARSER_ID_ALPHABET = ascii_letters + digits @classmethod def _generate_parser_id(cls, origin_id: str) -> str: - """Deterministic car-XXXX... (prefix 'car-' + 22 alphanumeric chars).""" + # Стабильный parser_id по origin_id. digest = hashlib.sha256(origin_id.encode()).digest() alphabet = cls._PARSER_ID_ALPHABET base = len(alphabet) @@ -363,7 +388,7 @@ class CarMapper: def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: # Формат: iaai:{lot_number} (аналог copart:94323985). - for value in [core.get("lot_number"), vehicle_summary.get("lot_number"), vehicle_summary.get("vin")]: + for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]: text = self._as_str(value) if text: return f"iaai:{text}" diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py index 359c451..5a3c244 100644 --- a/iaai_scraper/parsing/parser.py +++ b/iaai_scraper/parsing/parser.py @@ -4,16 +4,22 @@ import logging import re from typing import Any -from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty +from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty logger = logging.getLogger("iaai_scraper.parsers") class VehicleParser: - # DOM + JSON парсер страницы авто. + # Парсер данных страницы автомобиля. + + # Регулярные выражения для парсинга и детекции защиты. + _BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE) + _CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]) + _ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"]) + _CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE) + _ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE) SUMMARY_KEY_MAP = { - "vin": {"vin", "vehicleidentificationnumber"}, "lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"}, "year": {"year"}, "make": {"make", "manufacturer", "brand"}, @@ -32,10 +38,10 @@ class VehicleParser: "seller": {"seller", "sellername"}, "location": {"location", "branchname", "auctionlocation", "branch"}, "auction_date": {"auctiondate", "saledate", "liveauctiondate"}, - "body_type": {"bodytype", "bodystyle"}, - "drive": {"driveline", "drive"}, - "gearbox": {"transmission", "gearbox"}, - "engine": {"engine", "enginevolume"}, + "body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"}, + "drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"}, + "gearbox": {"transmission", "gearbox", "transmissiontype"}, + "engine": {"engine", "enginevolume", "enginetype", "enginedescription"}, "fuel_type": {"fueltype", "fuel"}, "cylinders": {"cylinders", "cylindercount"}, "color": {"color", "primarycolor", "exteriorcolor"}, @@ -43,24 +49,35 @@ class VehicleParser: DOM_LABEL_MAP: dict[str, str] = { "stock #": "lot_number", - "vin (status)": "vin", - "vin": "vin", + "stock": "lot_number", "primary damage": "primary_damage", "secondary damage": "secondary_damage", "odometer": "odometer", + "odometer (miles)": "odometer", + "mileage": "odometer", "body style": "body_type", + "body type": "body_type", + "vehicle type": "body_type", "engine": "engine", + "engine type": "engine", "transmission": "gearbox", "drive line type": "drive", + "driveline type": "drive", + "drive line": "drive", + "driveline": "drive", + "drive type": "drive", "fuel type": "fuel_type", + "fuel": "fuel_type", "cylinders": "cylinders", "exterior/interior": "color", "exterior color": "color", + "color": "color", "model": "model", "series": "trim", "selling branch": "location", "vehicle location": "vehicle_location", "auction date and time": "auction_date", + "sale date": "auction_date", "lane/run #": "lane", "actual cash value": "actual_cash_value", "estimated repair cost": "estimated_repair_cost", @@ -69,6 +86,7 @@ class VehicleParser: "title/sale doc brand": "title_brand", "start code": "run_and_drive", "key": "keys", + "keys": "keys", "manufactured in": "manufactured_in", "vehicle class": "vehicle_class", } @@ -79,15 +97,41 @@ class VehicleParser: return result lines = [line.strip() for line in dom_text.split("\n") if line.strip()] known_labels = set(self.DOM_LABEL_MAP.keys()) + skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"} + max_fields = len(set(self.DOM_LABEL_MAP.values())) + for i, line in enumerate(lines): - clean = line.rstrip(":").lower().strip() - if clean in known_labels and i + 1 < len(lines): + # Ранний выход, когда уже нашли все поля. + if len(result) >= max_fields: + break + + # Сценарий 1: "метка: значение" в одной строке. + colon_pos = line.find(":") + if colon_pos > 0: + label_part = line[:colon_pos].strip().lower() + value_part = line[colon_pos + 1:].strip() + if label_part in known_labels and value_part and value_part.lower() not in skip_values: + field_name = self.DOM_LABEL_MAP[label_part] + if field_name not in result or not result[field_name]: + result[field_name] = value_part + continue + + # Сценарий 2: метка и значение на соседних строках. + clean = line.rstrip(":").strip().lower() + clean_alt = clean.rstrip("#").strip() + matched_label = None + if clean in known_labels: + matched_label = clean + elif clean_alt in known_labels: + matched_label = clean_alt + + if matched_label and i + 1 < len(lines): value = lines[i + 1].strip() if value.rstrip(":").lower().strip() in known_labels: continue - if value.lower() in ("more actions", "view", "print", "share", "back to results"): + if value.lower() in skip_values: continue - field_name = self.DOM_LABEL_MAP[clean] + field_name = self.DOM_LABEL_MAP[matched_label] if field_name not in result or not result[field_name]: result[field_name] = value return result @@ -120,11 +164,11 @@ class VehicleParser: page_title = title_match.group(1).strip() title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) + # Один проход по payload для всех полей. + all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP) + summary: dict[str, Any] = {"source_url": vehicle_url} - for field, candidate_keys in self.SUMMARY_KEY_MAP.items(): - values: list[Any] = [] - for payload in payloads: - values.extend(deep_find_key(payload, candidate_keys)) + for field, values in all_found.items(): if field in dom_kv: values.append(dom_kv[field]) summary[field] = first_non_empty(values) @@ -133,11 +177,6 @@ class VehicleParser: summary["make"] = summary.get("make") or title_parsed.get("make") summary["model"] = summary.get("model") or title_parsed.get("model") summary["trim"] = summary.get("trim") or dom_kv.get("trim") - summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html) - if summary.get("vin") and isinstance(summary["vin"], str): - vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip() - vin_match = VIN_RE.search(vin_clean) - summary["vin"] = vin_match.group(1) if vin_match else vin_clean summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text) summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url) for dom_field, dom_value in dom_kv.items(): @@ -147,7 +186,7 @@ class VehicleParser: if not summary.get("actual_cash_value") and prices: summary["actual_cash_value"] = prices[0] if not summary.get("buy_now"): - buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE) + buy_now_match = self._BUY_NOW_RE.search(dom_text or "") if buy_now_match: summary["buy_now"] = buy_now_match.group(1) elif prices: @@ -160,25 +199,30 @@ class VehicleParser: p = item.get("payload") if isinstance(p, (dict, list)): payloads.append(p) - for field, candidate_keys in self.SUMMARY_KEY_MAP.items(): + # Дополнительный проход по встроенному JSON. + extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP) + for field, vals in extra.items(): if not summary.get(field): - val = first_non_empty(deep_find_key(p, candidate_keys)) - if val: - summary[field] = val + v = first_non_empty(vals) + if v: + summary[field] = v + # Передаём image_urls без повторного извлечения. + image_urls = summary.get("image_urls") or [] return { "vehicle_summary": summary, - "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url), + "payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls), "embedded_json": embedded, "dom_hints": self._dom_hints(dom_text), "access_notes": self._build_access_notes(summary, responses), } - def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]: - image_urls = self._extract_image_urls(payloads, "", vehicle_url) + def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]: + if image_urls is None: + image_urls = self._extract_image_urls(payloads, "", vehicle_url) return { "vehicle_core": { - "vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"), + "lot_number": summary.get("lot_number"), "year": summary.get("year"), "make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"), "odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"), "seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"), @@ -238,7 +282,6 @@ class VehicleParser: endpoints = [item.get("url", "") for item in responses] dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None)) return { - "vin_visible": bool(summary.get("vin")), "images_visible": bool(summary.get("image_urls")), "network_json_count": len(responses), "possible_captcha": bool(dom_hints.get("has_captcha_text")), @@ -266,11 +309,6 @@ class VehicleParser: return "JPY" return "USD" - @staticmethod - def _extract_vin(text: str) -> str | None: - match = VIN_RE.search(text or "") - return match.group(1) if match else None - @staticmethod def _extract_lot_number(text: str) -> str | None: match = LOT_RE.search(text or "") @@ -287,6 +325,9 @@ class VehicleParser: for script_text in scripts: if "{" not in script_text and "[" not in script_text: continue + # Пропускаем слишком большие минифицированные блоки. + if len(script_text) > 51_200: + continue try: parsed = json.loads(script_text.strip()) except Exception: @@ -361,8 +402,7 @@ class VehicleParser: "has_buy_now_text": "buy now" in lowered, "has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered, "has_damage_text": "damage" in lowered, - "has_vin_text": "vin" in lowered, "has_title_text": "title" in lowered, - "has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]), - "has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]), + "has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS), + "has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS), } diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py index d342d47..21c3778 100644 --- a/iaai_scraper/scraper.py +++ b/iaai_scraper/scraper.py @@ -5,8 +5,14 @@ import re import signal import time import uuid +import html as html_module +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen + +import urllib3 from playwright.sync_api import Error as PlaywrightError from playwright.sync_api import BrowserContext, Page, sync_playwright @@ -27,23 +33,51 @@ from .storage.schemas import CarRecord logger = logging.getLogger("iaai_scraper.scraper") VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) +HTML_TAG_RE = re.compile(r"<[^>]+>") +SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) class IAAIScraper: + @staticmethod + def _is_protection_or_network_error(exc: Exception) -> bool: + message = str(exc).lower() + signals = ( + "captcha", + "antibot", + "blocked", + "challenge", + "ns_error_net_interrupt", + "navigation", + "timeout", + "403", + "429", + ) + return any(signal in message for signal in signals) + + @staticmethod + def _html_to_text(html: str) -> str: + if not html: + return "" + cleaned = SCRIPT_STYLE_RE.sub(" ", html) + text = HTML_TAG_RE.sub(" ", cleaned) + text = html_module.unescape(text) + return re.sub(r"\s+", " ", text).strip() + @staticmethod def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None: dom_hints = parsed.get("dom_hints", {}) or {} access_notes = parsed.get("access_notes", {}) or {} summary = parsed.get("vehicle_summary", {}) or {} - if dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text"): + has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model")) + + if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity: raise AntiBotDetectedError(f"IAAI anti-bot detected for {vehicle_url}") - if access_notes.get("possible_captcha") or access_notes.get("possible_antibot"): + if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity: raise AntiBotDetectedError(f"IAAI blocked or challenged request for {vehicle_url}") - has_identity = bool(summary.get("lot_number") or summary.get("vin") or summary.get("make") or summary.get("model")) if not has_identity: raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") @@ -54,6 +88,21 @@ class IAAIScraper: return None return match.group(1) + @staticmethod + def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None: + raw_id = IAAIScraper._extract_origin_id_from_url(vehicle_url) + if not raw_id: + return None + return f"iaai:{raw_id}" + + @staticmethod + def _normalize_vehicle_url(vehicle_url: str) -> str: + try: + parts = urlsplit(vehicle_url) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + except Exception: + return vehicle_url + def __init__(self, runtime_settings: Settings | None = None) -> None: self.settings = runtime_settings or settings setup_logging(self.settings.log_level, self.settings.log_file) @@ -70,6 +119,29 @@ class IAAIScraper: self.car_mapper = CarMapper() self.persistence = PersistenceService(self.settings) self._shutdown_requested = False + # HTTP-клиент для fast-path. + # При наличии прокси используем ProxyManager. + proxy_url = self.settings.proxy.server + if proxy_url: + _proxy_kwargs: dict = { + "num_pools": 4, + "maxsize": 64, + "retries": False, + "timeout": urllib3.Timeout(connect=5, read=10), + } + if self.settings.proxy.username: + _proxy_kwargs["proxy_headers"] = urllib3.make_headers( + proxy_basic_auth=f"{self.settings.proxy.username}:{self.settings.proxy.password or ''}" + ) + self._http_pool: urllib3.PoolManager = urllib3.ProxyManager(proxy_url, **_proxy_kwargs) + logger.info("HTTP fast-path using proxy: %s", proxy_url) + else: + self._http_pool = urllib3.PoolManager( + num_pools=4, + maxsize=64, + retries=False, + timeout=urllib3.Timeout(connect=5, read=10), + ) def _new_trace_id(self, prefix: str) -> str: trace_id = f"{prefix}-{uuid.uuid4().hex[:8]}" @@ -90,7 +162,6 @@ class IAAIScraper: def close(self) -> None: if self.context is not None: try: - self._save_storage_state() self.context.close() except PlaywrightError: pass @@ -111,167 +182,192 @@ class IAAIScraper: finally: self.playwright = None - def _load_storage_state(self) -> str | None: - """Load saved browser session (cookies + localStorage) if valid.""" - session_cfg = self.settings.session - if not session_cfg.enabled: - return None - state_path = session_cfg.storage_state_path - if not state_path or not os.path.isfile(state_path): - logger.info("No saved session found at %s", state_path) - return None - try: - mtime = os.path.getmtime(state_path) - age_days = (time.time() - mtime) / 86400 - if age_days > session_cfg.max_age_days: - logger.info("Session expired (%.1f days old, max %d). Starting fresh.", age_days, session_cfg.max_age_days) - os.remove(state_path) - return None - logger.info("Reusing saved session from %s (%.1f days old)", state_path, age_days) - return state_path - except Exception as exc: - logger.warning("Failed to load session state: %s", exc) - return None - - def _save_storage_state(self) -> None: - """Save browser session (cookies + localStorage) to disk.""" - session_cfg = self.settings.session - if not session_cfg.enabled or not session_cfg.save_on_exit: - return - if not self.context: - return - state_path = session_cfg.storage_state_path - try: - Path(state_path).parent.mkdir(parents=True, exist_ok=True) - self.context.storage_state(path=state_path) - logger.info("Session state saved to %s", state_path) - except Exception as exc: - logger.warning("Failed to save session state: %s", exc) - - def _is_logged_in(self, page: Page) -> bool: - """Check if we are logged in by looking for account indicators.""" - try: - # IAAI shows user menu / 'My Account' / 'Sign Out' when logged in - logged_in = page.locator("a[href*='SignOut'], a[href*='signout'], .user-name, .welcome-user, [data-testid='user-menu']").count() > 0 - if not logged_in: - # Also check for Sign In link presence (means NOT logged in) - sign_in_visible = page.locator("a[href*='Login'], a[href*='login']").first.is_visible(timeout=3000) - return not sign_in_visible - return True - except Exception: - return False - - def _perform_login(self, page: Page) -> bool: - """Log in to IAAI using email/password credentials.""" - session_cfg = self.settings.session - if not session_cfg.has_credentials: - logger.warning("No IAAI credentials configured, skipping login") - return False - - logger.info("Logging in to IAAI as %s", session_cfg.login_email) - try: - page.goto(session_cfg.login_url, wait_until="domcontentloaded", timeout=60_000) - try: - page.wait_for_load_state("networkidle", timeout=15000) - except PlaywrightTimeoutError: - pass - time.sleep(2) - - # Fill email - email_input = page.locator("input[type='email'], input[name='Email'], input#Email, input[placeholder*='email' i], input[placeholder*='Email']").first - email_input.click() - time.sleep(0.5) - email_input.fill(session_cfg.login_email) - time.sleep(0.5) - - # Fill password - password_input = page.locator("input[type='password'], input[name='Password'], input#Password").first - password_input.click() - time.sleep(0.5) - password_input.fill(session_cfg.login_password) - time.sleep(1) - - # Click login button - login_btn = page.locator("button[type='submit'], input[type='submit'], button:has-text('Sign In'), button:has-text('Log In'), button:has-text('Login')").first - login_btn.click() - - # Wait for navigation after login - try: - page.wait_for_load_state("networkidle", timeout=20000) - except PlaywrightTimeoutError: - pass - time.sleep(3) - - # Check if login succeeded - current_url = page.url - if "login" not in current_url.lower(): - logger.info("Login successful, redirected to %s", current_url) - self._save_storage_state() - return True - - # Still on login page — check for errors - error_text = "" - try: - error_el = page.locator(".validation-summary-errors, .error-message, .alert-danger, [role='alert']").first - error_text = error_el.inner_text(timeout=3000) - except Exception: - pass - logger.error("Login failed. Still on login page. Error: %s", error_text or "unknown") - return False - - except Exception as exc: - logger.error("Login error: %s", exc) - return False - - def _ensure_logged_in(self) -> None: - """Ensure we have an authenticated session. Login if needed.""" - session_cfg = self.settings.session - if not session_cfg.has_credentials: - return - - # Check if saved session is still valid - page = self._get_page() - try: - page.goto(self.settings.home_url, wait_until="domcontentloaded", timeout=60_000) - try: - page.wait_for_load_state("networkidle", timeout=10000) - except PlaywrightTimeoutError: - pass - time.sleep(2) - - if self._is_logged_in(page): - logger.info("Already logged in, session is valid") - return - - logger.info("Not logged in, performing login...") - self._perform_login(page) - finally: - page.close() - - def _new_context(self, storage_state: str | None = None) -> BrowserContext: + def _new_context(self) -> BrowserContext: if self.browser is None: self.__enter__() if self.context: - self._save_storage_state() - self.context.close() - effective_state = storage_state or self._load_storage_state() - self.context = self.browser_factory.create_context(self.browser, storage_state=effective_state) + try: + self.context.close() + except PlaywrightError: + pass + self.context = self.browser_factory.create_context(self.browser) return self.context def init_db(self): self.persistence.create_tables() return {"status": "ok", "database_url": self.settings.database.url} - def _get_unauthenticated_page(self) -> Page: - context = self._new_context() - self._ensure_logged_in() - return context.new_page() + def _warmup_visit(self, page: Page) -> None: + # Прогрев главной страницы. + try: + logger.info("Warmup: visiting homepage to pass anti-bot challenge...") + page.goto(self.settings.home_url, wait_until="commit", timeout=30_000) + # Ждём domcontentloaded вместо networkidle. + try: + page.wait_for_load_state("domcontentloaded", timeout=6_000) + except PlaywrightTimeoutError: + pass + # Короткая проверка, что страница стабилизировалась. + try: + page.wait_for_function("() => document.title && document.title.length > 3", timeout=4_000) + except PlaywrightTimeoutError: + pass + # Короткая пауза для установки cookies. + time.sleep(0.3) + logger.info("Warmup done: %s (title=%s)", page.url, page.title()[:50]) + except Exception as e: + logger.warning("Warmup visit failed: %s — continuing anyway", e) + time.sleep(1.5) - def collect_listing(self, make: str | None = None, model: str | None = None): - page = self._get_unauthenticated_page() + def _get_page_with_warmup(self) -> Page: + context = self._new_context() + page = context.new_page() + # Прогрев с первой антибот-проверкой. + self._warmup_visit(page) + return page + + def _dedupe_urls(self, raw_urls: list[str]) -> list[str]: + # Нормализация и дедупликация URL. + vehicle_urls: list[str] = [] + seen: set[str] = set() + for raw in raw_urls: + normalized = self._normalize_vehicle_url(raw) + if normalized not in seen: + seen.add(normalized) + vehicle_urls.append(normalized) + return vehicle_urls + + def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: + # Отсев уже известных URL. + url_to_origin_id = { + url: self._extract_db_origin_id_from_url(url) + for url in vehicle_urls + } + candidate_origin_ids = [oid for oid in url_to_origin_id.values() if oid] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( + vehicle_urls, candidate_origin_ids, + ) + known_urls = { + url for url in vehicle_urls + if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) + } + skipped = len(known_urls) + if skipped: + logger.info("Filtering already known vehicles: skipped %d", skipped) + new_urls = [url for url in vehicle_urls if url not in known_urls] + return new_urls, skipped + + def _collect_listing_iterative( + self, + *, + make: str | None = None, + model: str | None = None, + limit: int, + ) -> tuple[list[str], list[str], dict, int]: + # Поэтапный сбор листинга до нужного лимита новых URL. + new_urls: list[str] = [] + all_raw_urls: list[str] = [] + seen: set[str] = set() + skipped_existing = 0 + pages_info: list[dict] = [] + max_pages = self.settings.listing.max_pages_per_run + + page = self._get_page_with_warmup() + try: + self.listing_collector.open_cars_listing(page) + self.listing_collector.apply_filters(page, make=make, model=model) + + for page_number in range(1, max_pages + 1): + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + pages_info.append({ + "page_number": page_result.page_number, + "links_found": len(page_result.vehicle_links), + }) + + # Собираем URL со страницы. + page_urls: list[str] = [] + for item in page_result.vehicle_links: + normalized = self._normalize_vehicle_url(item.href) + all_raw_urls.append(item.href) + if normalized not in seen: + seen.add(normalized) + page_urls.append(normalized) + + if not page_urls: + # Страница 1 пустая — скорее всего transient network issue. + # Пробуем перезагрузить листинг ещё раз. + if page_number == 1: + logger.warning("Page 1 returned 0 links — retrying listing open...") + time.sleep(3) + self.listing_collector.open_cars_listing(page) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + for item in page_result.vehicle_links: + normalized = self._normalize_vehicle_url(item.href) + all_raw_urls.append(item.href) + if normalized not in seen: + seen.add(normalized) + page_urls.append(normalized) + if not page_urls: + logger.info("Page %d: 0 new links, stopping pagination", page_number) + break + + # Фильтруем known. + fresh, page_skipped = self._filter_known_urls(page_urls) + skipped_existing += page_skipped + new_urls.extend(fresh) + + logger.info( + "Page %d: %d links, %d new, %d known (total new: %d/%d)", + page_number, len(page_urls), len(fresh), page_skipped, + len(new_urls), limit, + ) + + # Стопаем только если набрали нужное количество И на этой странице уже нет новых. + # Если последняя страница дала новые — проверяем следующую (там могут быть ещё). + if len(new_urls) >= limit and len(fresh) == 0: + break + if len(new_urls) >= limit: + # Набрали достаточно, дальше не листаем + break + + if not page_result.next_page_detected: + logger.info("No next page detected, stopping") + break + if not self.listing_collector.go_to_next_page(page): + logger.info("Failed to navigate to next page, stopping") + break + finally: + page.close() + + # Обрезаем до limit. + new_urls = new_urls[:limit] + + listing = { + "status": "ok", + "listing_url": self.settings.listing.cars_url, + "pages_collected": len(pages_info), + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": False, + "pages": pages_info, + } + return new_urls, all_raw_urls, listing, skipped_existing + + def collect_listing( + self, + make: str | None = None, + model: str | None = None, + known_origin_ids: set[str] | None = None, + ): + page = self._get_page_with_warmup() try: - listing = self.listing_collector.collect_listing_links(page, make=make, model=model) + listing = self.listing_collector.collect_listing_links( + page, + make=make, + model=model, + known_origin_ids=known_origin_ids, + ) finally: page.close() @@ -294,22 +390,200 @@ class IAAIScraper: finally: page.close() + # Быстрый извлекатель данных из inline JSON. + _JS_EXTRACT = """ + () => { + try { + const scripts = document.querySelectorAll('script:not([src])'); + for (const s of scripts) { + const t = s.textContent || ''; + if (!t.includes('inventoryView') || !t.includes('attributes')) continue; + const start = t.indexOf('{'); + if (start < 0) continue; + let depth = 0, end = -1; + for (let i = start; i < t.length; i++) { + if (t[i] === '{') depth++; + else if (t[i] === '}') { depth--; if (depth === 0) { end = i; break; } } + } + if (end < 0) continue; + try { + const obj = JSON.parse(t.slice(start, end + 1)); + const iv = obj.inventoryView; + if (!iv || !iv.attributes) continue; + const attr = iv.attributes; + const imgs = (iv.imageDimensions && iv.imageDimensions.keys && iv.imageDimensions.keys.$values) + ? iv.imageDimensions.keys.$values : []; + const bid = (obj.auctionInformation && obj.auctionInformation.biddingInformation) + ? obj.auctionInformation.biddingInformation : {}; + const prebid = (obj.auctionInformation && obj.auctionInformation.prebidInformation) + ? obj.auctionInformation.prebidInformation : {}; + return { + ok: true, + SalvageId: attr.SalvageId || '', + StockNumber: attr.StockNumber || '', + Year: attr.Year || '', + Make: attr.Make || '', + Model: attr.Model || '', + Series: attr.Series || '', + BodyStyleName: attr.BodyStyleName || '', + Cylinders: attr.Cylinders || '', + DriveLineTypeDesc: attr.DriveLineTypeDesc || '', + EngineSize: (attr.EngineInformation || attr.EngineSize || '').trim(), + FuelTypeCode: attr.FuelTypeCode || '', + Transmission: attr.Transmission || '', + ExteriorColor: attr.ExteriorColor || '', + PrimaryDamageDesc: attr.PrimaryDamageDesc || '', + SecondaryDamageDesc: attr.SecondaryDamageDesc || '', + ODOValue: attr.ODOValue || '', + ODOBrand: attr.ODOBrand || '', + RunAndDrive: attr.RunAndDrive || '', + Keys: attr.Keys || '', + BranchName: attr.BranchName || '', + AuctionDateTime: attr.AuctionDateTime || '', + Title: attr.Title || '', + TitleBrand: attr.TitleBrand || '', + TitleCode: attr.TitleCode || '', + EstRepairCost: attr.EstRepairCost || '', + VehicleGrade: attr.VehicleGrade || '', + highBidAmount: prebid.highBidAmount || bid.highBidAmount || '', + buyNowPrice: prebid.buyNowPrice || bid.buyNowPrice || '', + acv: attr.ProviderACV || '', + BranchNumber: attr.BranchNumber || '', + imageKeys: imgs.map(i => i.k || '').filter(Boolean), + }; + } catch (_) { continue; } + } + } catch (_) {} + return { ok: false }; + } + """ + + @staticmethod + def _build_car_from_js(js_data: dict, vehicle_url: str) -> dict: + # Сбор vehicle_summary из JS-данных. + if not js_data or not js_data.get("ok"): + return {} + + brnch = str(js_data.get("BranchNumber", "") or "").strip() + img_keys = js_data.get("imageKeys") or [] + image_urls = [ + f"https://vis.iaai.com/resizer?imageKeys={k}&width=845&height=633" + for k in img_keys + ] + + return { + "source_url": vehicle_url, + "lot_number": js_data.get("StockNumber") or js_data.get("SalvageId"), + "year": js_data.get("Year"), + "make": js_data.get("Make"), + "model": js_data.get("Model"), + "trim": js_data.get("Series"), + "body_type": js_data.get("BodyStyleName"), + "cylinders": js_data.get("Cylinders"), + "drive": js_data.get("DriveLineTypeDesc"), + "engine": js_data.get("EngineSize"), + "fuel_type": js_data.get("FuelTypeCode"), + "gearbox": js_data.get("Transmission"), + "color": js_data.get("ExteriorColor"), + "primary_damage": js_data.get("PrimaryDamageDesc"), + "secondary_damage": js_data.get("SecondaryDamageDesc"), + "odometer": js_data.get("ODOValue"), + "run_and_drive": js_data.get("RunAndDrive"), + "keys": js_data.get("Keys"), + "location": js_data.get("BranchName"), + "auction_date": js_data.get("AuctionDateTime"), + "title": js_data.get("Title"), + "current_bid": js_data.get("highBidAmount"), + "buy_now": js_data.get("buyNowPrice"), + "actual_cash_value": js_data.get("acv"), + "estimated_repair_cost": js_data.get("EstRepairCost"), + "image_urls": image_urls, + } + + @staticmethod + def _build_payload_insights(vehicle_summary: dict) -> dict: + # Сбор payload_insights из vehicle_summary. + return { + "vehicle_core": vehicle_summary, + "pricing": { + "buy_now": vehicle_summary.get("buy_now"), + "current_bid": vehicle_summary.get("current_bid"), + "actual_cash_value": vehicle_summary.get("actual_cash_value"), + "estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"), + "currency": "USD", + }, + "bids": {"amount": vehicle_summary.get("current_bid"), "currency": "USD"}, + "damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")}, + "auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")}, + "images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []}, + } + def _scrape_on_page(self, page: Page, vehicle_url: str): trace_id = self._new_trace_id("scrape") started_at = time.perf_counter() + + # Блокируем тяжёлые ресурсы страницы. + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + capture = NetworkCapture(self.settings) capture.attach(page, origin_url=vehicle_url) - page.goto(vehicle_url, wait_until="domcontentloaded", timeout=60_000) + page.goto(vehicle_url, wait_until="commit", timeout=60_000) + + # На VPS достаточно domcontentloaded. try: - page.wait_for_load_state("networkidle", timeout=10000) + page.wait_for_load_state("domcontentloaded", timeout=5_000) + except PlaywrightTimeoutError: + pass + + # Быстрый путь: читаем данные из inline JSON. + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + # Успешное извлечение структуры авто. + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + # Быстрая проверка валидности данных. + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError(f"JS extraction returned no vehicle identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + network_dump = capture.export() + result = { + "trace_id": trace_id, + "source_url": vehicle_url, + "fetched_at_epoch": int(time.time()), + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "network": network_dump, + "vehicle_summary": vehicle_summary, + "payload_insights": {}, + "embedded_json": [], + "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {}, + "db_record": db_record.model_dump(mode="json"), + } + if self.settings.raw_output_json: + save_to_json(network_dump, self.settings.raw_output_json) + return result + + # Резервный путь: полный парсинг страницы. + try: + page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=400) except PlaywrightTimeoutError: pass - time.sleep(self.settings.network_settle_ms / 1000) html = page.content() try: - dom_text = page.locator("body").inner_text(timeout=10_000) + dom_text = page.evaluate("() => document.body?.textContent || ''") except Exception: dom_text = "" network_dump = capture.export() @@ -336,8 +610,190 @@ class IAAIScraper: save_to_json(network_dump, self.settings.raw_output_json) return result + @staticmethod + def _extract_iaai_json_from_html(html: str, vehicle_url: str) -> dict | None: + """Извлекает IAAI inventoryView.attributes из HTML без браузера. + + Использует json.JSONDecoder.raw_decode для быстрого поиска JSON + вместо посимвольного сканирования скобок. + """ + search = "inventoryView" + pos = html.find(search) + if pos < 0: + return None + + # Ищем начало внешнего JSON-объекта. + script_start = html.rfind("", script_start) + if tag_end < 0: + return None + content_start = html.find("{", tag_end) + if content_start < 0: + return None + + # Быстрый поиск конца JSON через raw_decode. + decoder = json.JSONDecoder() + try: + obj, _ = decoder.raw_decode(html, content_start) + except (json.JSONDecodeError, ValueError): + return None + + iv = obj.get("inventoryView") + if not iv or not iv.get("attributes"): + return None + + attr = iv["attributes"] + imgs = [] + try: + imgs = iv.get("imageDimensions", {}).get("keys", {}).get("$values", []) or [] + except Exception: + pass + + bid = {} + prebid = {} + try: + ai = obj.get("auctionInformation", {}) + bid = ai.get("biddingInformation", {}) or {} + prebid = ai.get("prebidInformation", {}) or {} + except Exception: + pass + + img_keys = [i.get("k", "") for i in imgs if i.get("k")] + return { + "ok": True, + "SalvageId": attr.get("SalvageId", ""), + "StockNumber": attr.get("StockNumber", ""), + "Year": attr.get("Year", ""), + "Make": attr.get("Make", ""), + "Model": attr.get("Model", ""), + "Series": attr.get("Series", ""), + "BodyStyleName": attr.get("BodyStyleName", ""), + "Cylinders": attr.get("Cylinders", ""), + "DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""), + "EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(), + "FuelTypeCode": attr.get("FuelTypeCode", ""), + "Transmission": attr.get("Transmission", ""), + "ExteriorColor": attr.get("ExteriorColor", ""), + "PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""), + "SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""), + "ODOValue": attr.get("ODOValue", ""), + "ODOBrand": attr.get("ODOBrand", ""), + "RunAndDrive": attr.get("RunAndDrive", ""), + "Keys": attr.get("Keys", ""), + "BranchName": attr.get("BranchName", ""), + "AuctionDateTime": attr.get("AuctionDateTime", ""), + "Title": attr.get("Title", ""), + "TitleBrand": attr.get("TitleBrand", ""), + "TitleCode": attr.get("TitleCode", ""), + "EstRepairCost": attr.get("EstRepairCost", ""), + "VehicleGrade": attr.get("VehicleGrade", ""), + "highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""), + "buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""), + "acv": attr.get("ProviderACV", ""), + "BranchNumber": attr.get("BranchNumber", ""), + "imageKeys": img_keys, + } + + def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord: + # Быстрый запрос через context.request. + if not self.context: + self._new_context() + assert self.context is not None + last_error: Exception | None = None + max_attempts = max(1, self.settings.fast_path_max_attempts) + timeout_ms = max(1000, self.settings.fast_path_timeout_ms) + for attempt in range(1, max_attempts + 1): + try: + response = self.context.request.get(vehicle_url, timeout=timeout_ms) + if not response.ok: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + + html = response.text() + + # Пытаемся извлечь JSON напрямую из HTML. + js_data = self._extract_iaai_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + except Exception as exc: + last_error = exc + if attempt < max_attempts: + time.sleep(0.2) + + if last_error is not None: + raise last_error + raise RuntimeError(f"HTTP fast-path failed for {vehicle_url}") + + def _cookie_header_for_context(self) -> str: + if not self.context: + return "" + try: + cookies = self.context.cookies() + except Exception: + return "" + pairs = [ + f"{item.get('name')}={item.get('value')}" + for item in cookies + if item.get("name") and item.get("value") is not None + ] + return "; ".join(pairs) + + def _scrape_via_raw_http( + self, + vehicle_url: str, + *, + cookie_header: str, + user_agent: str, + ) -> CarRecord: + """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive).""" + headers = { + "User-Agent": user_agent, + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + } + if cookie_header: + headers["Cookie"] = cookie_header + + response = self._http_pool.request("GET", vehicle_url, headers=headers) + if response.status >= 400: + raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") + html = response.data.decode("utf-8", errors="ignore") + + js_data = self._extract_iaai_json_from_html(html, vehicle_url) + if not js_data: + raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") + + vehicle_summary = self._build_car_from_js(js_data, vehicle_url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise RuntimeError(f"HTTP fast-path returned incomplete identity for {vehicle_url}") + + db_record = self.car_mapper.map_to_car_record( + vehicle_url=vehicle_url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + return CarRecord.model_validate(db_record.model_dump(mode="json")) + def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"): - # Scrape + upsert одного авто. + # Скрапинг и upsert одного авто. trace_id = self._new_trace_id("sync-vehicle") started_at = time.perf_counter() self.persistence.create_tables() @@ -384,6 +840,290 @@ class IAAIScraper: error_summary=error_summary, ) + # ── Tunables for sync_batch ── + _HTTP_MICRO_BATCH = 25 # URLs per micro-batch (avoid mass rate-limit) + _HTTP_MAX_RETRIES = 2 # Retries per URL before giving up to browser + _HTTP_RETRY_DELAYS = (0.4, 1.0) # Backoff between retries + _FALLBACK_PARALLEL_PAGES = 4 # Concurrent browser tabs for fallback + _FALLBACK_NAV_TIMEOUT_MS = 8000 # Reduced from 15 000 + + def _browser_fallback_parallel( + self, + fallback_urls: list[tuple[str, int]], + total: int, + n_pages: int, + ) -> dict: + # Обработка fallback URL через браузер. + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + cars_failed = 0 + protection_events = 0 + + # Деление URL по страницам. + slices: list[list[tuple[str, int]]] = [[] for _ in range(n_pages)] + for i, item in enumerate(fallback_urls): + slices[i % n_pages].append(item) + + def _process_slice(url_slice: list[tuple[str, int]]) -> dict: + local_records: list[CarRecord] = [] + local_failures: list[dict[str, str]] = [] + local_failed = 0 + local_protection = 0 + page: Page | None = None + try: + for url, global_idx in url_slice: + if page is None: + page = self._get_page() + if self.settings.block_resources: + BrowserFactory.enable_resource_blocking(page) + + try: + page.goto( + url, + wait_until="commit", + timeout=self._FALLBACK_NAV_TIMEOUT_MS, + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed to open %s: %s", global_idx, total, url, exc) + try: + page.close() + except Exception: + pass + page = None + continue + + try: + js_data: dict = {} + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if not js_data.get("ok"): + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except PlaywrightTimeoutError: + pass + try: + js_data = page.evaluate(self._JS_EXTRACT) or {} + except Exception: + pass + + if js_data.get("ok"): + vehicle_summary = self._build_car_from_js(js_data, url) + has_identity = bool(vehicle_summary.get("make") or vehicle_summary.get("lot_number")) + if not has_identity: + raise SiteStructureChangedError( + f"JS extraction returned no vehicle identity for {url}" + ) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=vehicle_summary, + payload_insights=self._build_payload_insights(vehicle_summary), + ) + else: + try: + page.wait_for_selector( + "#VehicleDetailViewModel, .veh-details, .vehicle-details, " + "[data-uname='vehicleDetailPage']", + timeout=400, + ) + except PlaywrightTimeoutError: + pass + page_html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = "" + network_dump = { + "requests": [], + "json_responses": [], + "capture_limits": {}, + } + parsed = self.vehicle_parser.normalize(url, page_html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, url) + db_record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=parsed.get("vehicle_summary", {}), + payload_insights=parsed.get("payload_insights", {}), + ) + + record = CarRecord.model_validate(db_record.model_dump(mode="json")) + local_records.append(record) + logger.debug( + "[%d/%d] Parsed %s %s %s (fallback)", + global_idx, total, record.brand, record.model, record.year or "?", + ) + except Exception as exc: + if self._is_protection_or_network_error(exc): + local_protection += 1 + local_failed += 1 + local_failures.append({"vehicle_url": url, "error": str(exc)}) + logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) + finally: + if page is not None: + try: + page.close() + except Exception: + pass + return { + "records": local_records, + "failures": local_failures, + "cars_failed": local_failed, + "protection_events": local_protection, + } + + # Один page обрабатываем в главном потоке. + if n_pages == 1: + res = _process_slice(slices[0] if slices else []) + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + else: + # Несколько страниц — параллельно через потоки (каждый поток со своей Page). + with ThreadPoolExecutor(max_workers=n_pages) as executor: + futures = [executor.submit(_process_slice, s) for s in slices if s] + for fut in as_completed(futures): + res = fut.result() + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] + + return { + "records": records, + "failures": failures, + "cars_failed": cars_failed, + "protection_events": protection_events, + } + + def sync_batch( + self, + vehicle_urls: list[str], + lane: str = "iaai_cars", + parallel_tabs: int | None = None, + ) -> dict: + # Пакетный скрапинг списка URL. + trace_id = self._new_trace_id("sync-batch") + started_at = time.perf_counter() + num_workers = parallel_tabs or self.settings.parallel_tabs + num_workers = min(num_workers, len(vehicle_urls), 48) + + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + records: list[CarRecord] = [] + failures: list[dict[str, str]] = [] + http_successes = 0 + http_fallbacks = 0 + protection_events = 0 + + total = len(vehicle_urls) + logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) + + # Подготовка для HTTP fast-path: cookies + user-agent из Playwright сессии. + cookie_header = self._cookie_header_for_context() + user_agent = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" + ) + + # ── Phase 1: HTTP fast-path — все URLs одним ThreadPoolExecutor ── + fallback_urls: list[tuple[str, int]] = [] + + def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: + idx, url = idx_url + last_exc: Exception | None = None + for attempt in range(1 + self._HTTP_MAX_RETRIES): + try: + return idx, self._scrape_via_raw_http( + url, + cookie_header=cookie_header, + user_agent=user_agent, + ) + except Exception as exc: + last_exc = exc + if attempt < self._HTTP_MAX_RETRIES: + time.sleep(self._HTTP_RETRY_DELAYS[min(attempt, len(self._HTTP_RETRY_DELAYS) - 1)]) + return idx, last_exc # type: ignore[return-value] + + # Запускаем все URLs сразу — один пул потоков для минимального времени ожидания. + indexed_urls = list(enumerate(vehicle_urls)) + with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor: + for idx, result in executor.map(_fetch_one_with_retry, indexed_urls): + if isinstance(result, Exception): + http_fallbacks += 1 + logger.debug( + "[%d/%d] HTTP fast-path failed for %s: %s", + idx + 1, total, vehicle_urls[idx], result, + ) + fallback_urls.append((vehicle_urls[idx], idx + 1)) + else: + records.append(result) + http_successes += 1 + logger.debug( + "[%d/%d] Parsed %s %s %s (raw HTTP)", + idx + 1, total, result.brand, result.model, result.year or "?", + ) + + logger.info( + "HTTP phase done: %d OK, %d fallback (%.1fs)", + http_successes, http_fallbacks, time.perf_counter() - started_at, + ) + + # ── Phase 2: browser fallback (single-threaded — Playwright sync API is not thread-safe) ── + if fallback_urls: + logger.info( + "Fallback browser mode: %d/%d URLs, single page", + len(fallback_urls), total, + ) + fb_results = self._browser_fallback_parallel(fallback_urls, total, 1) + records.extend(fb_results["records"]) + cars_failed += fb_results["cars_failed"] + protection_events += fb_results["protection_events"] + failures.extend(fb_results["failures"]) + + # ── Phase 3: single DB flush ── + if records: + # Дедупликация перед записью. + seen_origins: set[str] = set() + unique_records: list[CarRecord] = [] + for rec in records: + key = rec.origin_id or rec.origin_url + if key not in seen_origins: + seen_origins.add(key) + unique_records.append(rec) + if len(unique_records) < len(records): + logger.info("Dedup before DB: %d → %d", len(records), len(unique_records)) + records = unique_records + + try: + batch_result = self.persistence.upsert_cars_batch(records) + cars_upserted = batch_result["inserted"] + batch_result["updated"] + images_upserted = batch_result["images_upserted"] + except Exception as exc: + logger.error("Batch upsert failed: %s", exc) + cars_failed += len(records) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "trace_id": trace_id, + "status": status, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "http_successes": http_successes, + "http_fallbacks": http_fallbacks, + "protection_events": protection_events, + "elapsed_seconds": round(time.perf_counter() - started_at, 3), + "failures": failures, + } + def sync_listing( self, make: str | None = None, @@ -416,94 +1156,107 @@ class IAAIScraper: listing: dict = {} try: - listing = self.collect_listing(make=make, model=model) - vehicle_urls = list(listing.get("vehicle_urls", [])) - effective_only_new = self.settings.sync_only_new if only_new is None else only_new - if effective_only_new: - existing_urls = self.persistence.get_existing_origin_urls(vehicle_urls) - url_to_origin_id = { - url: self._extract_origin_id_from_url(url) - for url in vehicle_urls - } - candidate_origin_ids = [origin_id for origin_id in url_to_origin_id.values() if origin_id] - existing_ids = self.persistence.get_existing_origin_ids(candidate_origin_ids) - known_urls = { - url - for url in vehicle_urls - if (url in existing_urls) or (url_to_origin_id.get(url) in existing_ids) - } + # ── Сбор листинга ── + # При only_new + limit используем итеративный подход: + # листаем страницы одну за другой, фильтруем known на лету, + # останавливаемся когда набрали limit новых. + original_listing_cap = self.settings.listing.max_vehicles_per_run + original_include_pagination = self.settings.listing.include_pagination + original_collect_current_page_only = self.settings.listing.collect_current_page_only + original_max_pages_per_run = self.settings.listing.max_pages_per_run - skipped_existing = len(known_urls) - if skipped_existing: - logger.info("Filtering already known vehicles: skipped %d", skipped_existing) - vehicle_urls = [url for url in vehicle_urls if url not in known_urls] + if effective_only_new and limit is not None and limit > 0: + # Итеративный сбор: страница→фильтр→проверка→следующая страница. + vehicle_urls, raw_urls, listing, skipped_existing = self._collect_listing_iterative( + make=make, model=model, limit=limit, + ) + else: + # Обычный сбор (без only_new или без limit). + prefetch_cap: int | None = None + if limit is not None and limit > 0: + prefetch_cap = limit + if prefetch_cap < original_listing_cap: + self.settings.listing.max_vehicles_per_run = prefetch_cap - if limit is not None: - vehicle_urls = vehicle_urls[:max(0, limit)] + # Если включён режим только новых — заранее загружаем все известные origin_id, + # чтобы listing_collector мог остановиться при встрече старых страниц. + known_origin_ids: set[str] | None = None + if effective_only_new and self.settings.listing.early_stop_threshold > 0.0: + try: + known_origin_ids = self.persistence.get_all_origin_ids_for_lane("iaai:") + logger.info( + "Loaded %d known origin_ids for early-stop listing", + len(known_origin_ids), + ) + except Exception as exc: + logger.warning("Could not load known origin_ids for early-stop: %s", exc) + + try: + listing = self.collect_listing( + make=make, + model=model, + known_origin_ids=known_origin_ids, + ) + finally: + self.settings.listing.max_vehicles_per_run = original_listing_cap + self.settings.listing.include_pagination = original_include_pagination + self.settings.listing.collect_current_page_only = original_collect_current_page_only + self.settings.listing.max_pages_per_run = original_max_pages_per_run + + raw_urls = list(listing.get("vehicle_urls", [])) + vehicle_urls = self._dedupe_urls(raw_urls) + + if effective_only_new: + vehicle_urls, skipped_existing = self._filter_known_urls(vehicle_urls) + + if limit is not None: + vehicle_urls = vehicle_urls[:max(0, limit)] total = len(vehicle_urls) - logger.info("Starting sync: %d vehicles to process", total) + logger.info("Starting sync: %d vehicles to process (batch mode)", total) - for index, vehicle_url in enumerate(vehicle_urls, start=1): - page = self._get_page() + # Собираем нормализованные origin_url для последующей пометки проданных. + # Используем URL (а не origin_id из URL), т.к. в БД origin_id берётся из parsed lot_number, + # который может отличаться от числа в URL листинга. + all_listing_origin_urls = set() + for raw_url in raw_urls: + normalized_url = self._normalize_vehicle_url(raw_url) + if normalized_url: + all_listing_origin_urls.add(normalized_url) + + # Обрабатываем пакетами. + batch_size = self.settings.celery.batch_size + for batch_start in range(0, total, batch_size): + batch_urls = vehicle_urls[batch_start:batch_start + batch_size] + logger.info( + "Processing batch %d-%d of %d", + batch_start + 1, min(batch_start + batch_size, total), total, + ) + batch_result = self.sync_batch(batch_urls, lane=lane) + cars_upserted += batch_result.get("cars_upserted", 0) + cars_failed += batch_result.get("cars_failed", 0) + images_upserted += batch_result.get("images_upserted", 0) + failures.extend(batch_result.get("failures", [])) + + # Помечаем авто как проданные, если они исчезли из листинга. + # Только если сканирование было полным (не ограниченным limit/only_new/early_stop). + is_partial_scan = ( + effective_only_new + or (limit is not None and limit > 0) + or listing.get("early_stopped", False) + ) + if all_listing_origin_urls and not is_partial_scan: try: - logger.info("[%d/%d] Scraping %s", index, total, vehicle_url) - scrape_result = self._scrape_on_page(page, vehicle_url) - db_record = scrape_result.get("db_record") - if not db_record: - raise RuntimeError("Scrape result does not contain db_record") - record = CarRecord.model_validate(db_record) - - # Применяем фильтры из runtime_config (include/exclude/price/mileage/flags) - if not self.runtime_config.filters.is_empty(): - vehicle_summary = scrape_result.get("vehicle_summary", {}) or {} - filter_values = { - "brand": record.brand, - "model": record.model, - "year": record.year, - "body_type": record.body_type, - "color": record.color, - "drive": record.drive, - "gearbox": record.gearbox, - "location": vehicle_summary.get("location"), - "price": record.price, - "mileage": record.mileage, - "is_damaged": record.is_damaged, - "run_and_drive": vehicle_summary.get("run_and_drive"), - } - if not self.runtime_config.filters.matches(filter_values): - logger.info( - "[%d/%d] Skipped by filter: %s %s %s", - index, total, record.brand, record.model, record.year or "?", - ) - cars_filtered += 1 - continue - - upsert = self.persistence.upsert_car(record) - if upsert.get("action") != "skipped": - cars_upserted += 1 - img_count = int(upsert.get("images_upserted", 0)) - images_upserted += img_count - logger.info( - "[%d/%d] %s %s: %s %s %s — %s, %d images", - index, total, upsert.get("action", "?"), - record.origin_id, record.brand, record.model, - record.year or "?", record.price or "N/A", img_count, - ) + sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) + if sold_count: + logger.info("Marked %d cars as sold", sold_count) except Exception as exc: - cars_failed += 1 - failures.append({"vehicle_url": vehicle_url, "error": str(exc)}) - logger.error("[%d/%d] Failed %s: %s", index, total, vehicle_url, exc) - finally: - try: - page.close() - except Exception: - pass - - if index < total: - self.pacer.between_vehicles() + logger.warning("Failed to mark sold cars: %s", exc) + elif is_partial_scan: + logger.debug("Skipping mark_sold: partial/incremental scan (only_new=%s, limit=%s, early_stopped=%s)", + effective_only_new, limit, listing.get("early_stopped", False)) except Exception as exc: if not failures: failures.append({"vehicle_url": "collect_listing", "error": str(exc)}) @@ -540,7 +1293,7 @@ class IAAIScraper: } def run_scheduled(self) -> None: - # NOTE: Зарезервировано для standalone-режима (без Celery beat). + # Резерв для standalone-режима. # В текущей архитектуре планирование выполняется через Celery beat + worker/tasks.py. def _handle_shutdown(signum, frame): logger.info("Received signal %s, shutting down gracefully...", signum) diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py index 1f30820..22e6fda 100644 --- a/iaai_scraper/storage/db.py +++ b/iaai_scraper/storage/db.py @@ -1,9 +1,10 @@ import logging from contextlib import contextmanager from datetime import datetime, timezone -from typing import Iterator +from typing import Any, Iterator -from sqlalchemy import create_engine, or_, select +from sqlalchemy import create_engine, delete, or_, select, text, update +from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session, sessionmaker from ..core.config import Settings @@ -18,6 +19,8 @@ CAR_DB_FIELDS = { if col.key not in ("id",) } +_IN_CHUNK_SIZE = 5000 + class PersistenceService: @@ -99,21 +102,250 @@ class PersistenceService: rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all() return {str(row[0]) for row in rows if row and row[0]} + def get_existing_urls_and_ids( + self, origin_urls: list[str], origin_ids: list[str], + ) -> tuple[set[str], set[str]]: + # Загрузка существующих URL и origin_id. + if not origin_urls and not origin_ids: + return set(), set() + urls: set[str] = set() + ids: set[str] = set() + with self.session_scope() as session: + max_len = max(len(origin_urls), len(origin_ids), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + conditions = [] + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car.origin_url, Car.origin_id).where(or_(*conditions)) + ).all() + for r in rows: + if r[0]: + urls.add(str(r[0])) + if r[1]: + ids.add(str(r[1])) + return urls, ids + @staticmethod def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: - for image_payload in images: - session.add(Image(fullres_image=str(image_payload["fullres_image"]), preview_image=str(image_payload["preview_image"]), order_index=int(image_payload.get("order_index", 0)), car_id=car_id)) + if not images: + return + session.add_all([ + Image( + fullres_image=str(img["fullres_image"]), + preview_image=str(img["preview_image"]), + order_index=int(img.get("order_index", 0)), + car_id=car_id, + ) + for img in images + ]) @staticmethod def _car_payload(record: CarRecord) -> dict[str, object]: payload = record.model_dump(mode="python") return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + def _is_postgres(self) -> bool: + return self.engine.dialect.name == "postgresql" + + def _load_existing_cars( + self, + session: Session, + origin_ids: list[str], + origin_urls: list[str], + ) -> tuple[dict[str, Car], dict[str, Car]]: + existing_by_id: dict[str, Car] = {} + existing_by_url: dict[str, Car] = {} + if not origin_ids and not origin_urls: + return existing_by_id, existing_by_url + + existing_cars: list[Car] = [] + max_len = max(len(origin_ids), len(origin_urls), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + conditions = [] + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car).where(or_(*conditions)) + ).scalars().all() + existing_cars.extend(rows) + + for car in existing_cars: + if car.origin_id: + existing_by_id[car.origin_id] = car + if car.origin_url: + existing_by_url[car.origin_url] = car + return existing_by_id, existing_by_url + + def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]: + existing_images_map: dict[int, set[str]] = {} + if not car_ids: + return existing_images_map + + car_id_list = list(car_ids) + for i in range(0, len(car_id_list), _IN_CHUNK_SIZE): + chunk = car_id_list[i:i + _IN_CHUNK_SIZE] + img_rows = session.execute( + select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk)) + ).all() + for cid, furl in img_rows: + existing_images_map.setdefault(int(cid), set()).add(str(furl)) + return existing_images_map + + @staticmethod + def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]: + return { + key: getattr(insert_stmt.excluded, key) + for key in CAR_DB_FIELDS + } + + def _replace_images_for_car( + self, + session: Session, + car_id: int, + images: list[dict[str, object]], + origin_id: str, + ) -> int: + nested = session.begin_nested() + try: + session.execute(delete(Image).where(Image.car_id == car_id)) + self._add_images(session, car_id, images) + session.flush() + nested.commit() + return len(images) + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True) + return 0 + + def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]: + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + entries: list[dict[str, object]] = [] + upsert_payloads: list[dict[str, object]] = [] + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + car_by_id = existing_by_id.get(record.origin_id) + car_by_url = existing_by_url.get(record.origin_url) + entry: dict[str, object] = {"record": record, "images": images, "car_id": None} + + if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + entry["car_id"] = int(car_by_url.id) + updated += 1 + else: + upsert_payloads.append(payload) + if car_by_id is not None: + updated += 1 + else: + inserted += 1 + entries.append(entry) + + session.flush() + + if upsert_payloads: + insert_stmt = pg_insert(Car).values(upsert_payloads) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id, Car.origin_id) + rows = session.execute(upsert_stmt).all() + car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows} + for entry in entries: + if entry["car_id"] is not None: + continue + record = entry["record"] + car_id = car_ids_by_origin_id.get(record.origin_id) + if car_id is None: + raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}") + entry["car_id"] = car_id + + car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None} + existing_images_map = self._load_existing_image_urls(session, car_ids) + images_by_car_id: dict[int, list[dict[str, object]]] = {} + replace_ids: list[int] = [] + + for entry in entries: + car_id = int(entry["car_id"]) + images = entry["images"] + new_image_urls = { + str(img.get("fullres_image", "")) + for img in images + if img.get("fullres_image") + } + old_image_urls = existing_images_map.get(car_id, set()) + if new_image_urls != old_image_urls: + replace_ids.append(car_id) + images_by_car_id[car_id] = images + else: + images_total += len(old_image_urls) + + if replace_ids: + for i in range(0, len(replace_ids), _IN_CHUNK_SIZE): + chunk = replace_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car_id in replace_ids: + images = images_by_car_id[car_id] + self._add_images(session, car_id, images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + def upsert_car(self, record: CarRecord): - # Insert/update автомобиля по origin_id/origin_url. + # Вставка или обновление автомобиля по origin_id/origin_url. payload = self._car_payload(record) images = [image.model_dump(mode="python") for image in record.images] with self.session_scope() as session: + if self._is_postgres(): + car_by_url = session.execute( + select(Car).where(Car.origin_url == record.origin_url) + ).scalar_one_or_none() + if car_by_url is not None and car_by_url.origin_id != record.origin_id: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + session.flush() + car_id = int(car_by_url.id) + action = "updated" + else: + existed = session.execute( + select(Car.id).where(Car.origin_id == record.origin_id) + ).scalar_one_or_none() is not None + insert_stmt = pg_insert(Car).values(**payload) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id) + car_id = int(session.execute(upsert_stmt).scalar_one()) + action = "updated" if existed or car_by_url is not None else "inserted" + + images_upserted = self._replace_images_for_car( + session, car_id, images, record.origin_id, + ) + return {"car_id": car_id, "images_upserted": images_upserted, "action": action} + car = session.execute( select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) ).scalar_one_or_none() @@ -128,19 +360,217 @@ class PersistenceService: setattr(car, key, value) car.last_seen_at = record.last_seen_at session.flush() - nested = session.begin_nested() - try: - for image in list(car.images): - session.delete(image) - session.flush() - self._add_images(session, int(car.id), images) - session.flush() - nested.commit() - except Exception: - nested.rollback() - logger.warning("Image replacement failed for car %s, keeping old images", record.origin_id) - images = [] - return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id) + return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action} self._add_images(session, int(car.id), images) session.flush() return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]: + """Пакетный upsert нескольких автомобилей в одной транзакции. + + Оптимизации: + - Дедупликация записей по origin_id перед вставкой. + - Chunked IN-queries для больших списков (обход лимита PG параметров). + - Пропуск перезаписи изображений, если набор URL не изменился. + - Один DELETE по car_id IN (...) вместо удаления по одному. + - Fallback на по-одному upsert если batch commit упал. + """ + # ── Дедупликация записей внутри батча ── + seen_ids: dict[str, int] = {} + unique_records: list[CarRecord] = [] + for idx, r in enumerate(records): + key = r.origin_id or r.origin_url + if key in seen_ids: + logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key]) + continue + seen_ids[key] = idx + unique_records.append(r) + + if len(unique_records) < len(records): + logger.info("Deduped batch: %d → %d records", len(records), len(unique_records)) + records = unique_records + + try: + return self._upsert_cars_batch_inner(records) + except Exception as exc: + logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc) + return self._upsert_cars_individually(records) + + def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]: + """Внутренняя реализация batched upsert (одна транзакция).""" + if self._is_postgres(): + return self._upsert_cars_batch_postgres(records) + + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + # Получаем существующие записи chunked-запросами. + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + # Предзагружаем ВСЕ изображения для обновляемых машин одним запросом. + existing_car_ids = set() + for record in records: + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is not None: + existing_car_ids.add(int(car.id)) + + # Строим маппинг car_id → set(image_urls) для сравнения. + existing_images_map = self._load_existing_image_urls(session, existing_car_ids) + + new_cars: list[tuple[Car, list[dict]]] = [] + update_cars_needing_images: list[tuple[Car, list[dict]]] = [] + + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is None: + car = Car(**payload) + session.add(car) + inserted += 1 + new_cars.append((car, images)) + else: + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + updated += 1 + + # Проверяем, изменились ли изображения. + new_image_urls = {img.get("fullres_image", "") for img in images} + old_image_urls = existing_images_map.get(int(car.id), set()) + if new_image_urls != old_image_urls: + update_cars_needing_images.append((car, images)) + else: + images_total += len(old_image_urls) + + # Один flush для всех вставок. + session.flush() + + # Добавляем изображения для новых автомобилей. + for car, images in new_cars: + self._add_images(session, int(car.id), images) + images_total += len(images) + + # Массово обновляем изображения только для машин с изменёнными картинками. + if update_cars_needing_images: + update_ids = [int(car.id) for car, _ in update_cars_needing_images] + for i in range(0, len(update_ids), _IN_CHUNK_SIZE): + chunk = update_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car, images in update_cars_needing_images: + self._add_images(session, int(car.id), images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]: + # Fallback на поштучный upsert. + inserted = 0 + updated = 0 + images_total = 0 + for record in records: + try: + result = self.upsert_car(record) + action = result.get("action", "inserted") + if action == "inserted": + inserted += 1 + else: + updated += 1 + images_total += int(result.get("images_upserted", 0)) + except Exception as exc: + logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int: + """Помечает авто как проданные, если их нет в активном листинге (по origin_id).""" + if not active_origin_ids: + return 0 + with self.session_scope() as session: + stmt = ( + update(Car) + .where(Car.origin_id.notin_(active_origin_ids)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("iaai:%")) + .values(is_sold=True) + ) + result = session.execute(stmt) + count = result.rowcount or 0 + if count: + logger.info("Marked %d cars as sold (no longer in listing)", count) + return count + + def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int: + """Помечает авто как проданные, если их URL нет в активном листинге. + + Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, + что кардинально быстрее при больших объёмах (100K+ URLs). + """ + if not active_origin_urls: + return 0 + + is_postgres = "postgresql" in self.settings.database.url + + with self.session_scope() as session: + if is_postgres: + # Создаём временную таблицу с активными URL. + session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP")) + session.execute(text("TRUNCATE _active_urls")) + + # Вставляем активные URL чанками. + url_list = list(active_origin_urls) + for i in range(0, len(url_list), _IN_CHUNK_SIZE): + chunk = url_list[i:i + _IN_CHUNK_SIZE] + values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) + params = {f"u{j}": url for j, url in enumerate(chunk)} + session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params) + + # Создаём индекс на временной таблице для ускорения JOIN. + session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)")) + + # Массовая пометка проданных в PostgreSQL. + result = session.execute(text(""" + UPDATE cars + SET is_sold = TRUE + FROM ( + SELECT c.id + FROM cars c + LEFT JOIN _active_urls a ON c.origin_url = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'iaai:%%' + ) sub + WHERE cars.id = sub.id + """)) + count = result.rowcount or 0 + else: + # Упрощённый путь для SQLite. + stmt = ( + update(Car) + .where(Car.origin_url.notin_(active_origin_urls)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("iaai:%")) + .values(is_sold=True) + ) + result = session.execute(stmt) + count = result.rowcount or 0 + + if count: + logger.info("Marked %d cars as sold by URL (no longer in listing)", count) + return count + + def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]: + """Возвращает все известные origin_id для заданного префикса. + + Использует yield_per для потоковой загрузки при большом количестве записей. + """ + with self.session_scope() as session: + result = session.execute( + select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} \ No newline at end of file diff --git a/iaai_scraper/storage/enums.py b/iaai_scraper/storage/enums.py index 59138e6..d25d8a5 100644 --- a/iaai_scraper/storage/enums.py +++ b/iaai_scraper/storage/enums.py @@ -18,15 +18,6 @@ BODY_TYPE_ENUM_VALUES = ( ) COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") ORIGIN_ENUM_VALUES = ( - "TAU", - "CARSENSOR", - "HANAMARU", - "ENCAR", - "KURUMA_TRADER", - "ASNET", - "KABABA", - "ACV", - "COPART", "IAAI", "NA", ) diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py index 3fd5eaf..29d5566 100644 --- a/iaai_scraper/storage/models.py +++ b/iaai_scraper/storage/models.py @@ -23,6 +23,7 @@ class Car(Base): __tablename__ = "cars" __table_args__ = ( Index("ix_cars_brand_model", "brand", "model"), + Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"), ) id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) @@ -30,21 +31,21 @@ class Car(Base): model: Mapped[str] = mapped_column(String(50), nullable=False) year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) - currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=True, create_constraint=False), nullable=False, default="USD") + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD") mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA") is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) color: Mapped[str] = mapped_column(String(), nullable=False, default="other") - drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=True, create_constraint=False), nullable=True) - gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=True, create_constraint=False), nullable=True) - steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=True, create_constraint=False), nullable=True) - body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=True, create_constraint=False), nullable=False, default="OTHER") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER") engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) - selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA") one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=True, create_constraint=False), nullable=False, default="NA") + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA") origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) diff --git a/iaai_scraper/worker/tasks.py b/iaai_scraper/worker/tasks.py index b11bf5f..e244f3a 100644 --- a/iaai_scraper/worker/tasks.py +++ b/iaai_scraper/worker/tasks.py @@ -1,8 +1,10 @@ -# Celery-задачи для синхронизации автомобилей и листинга IAAI. +# Задачи Celery для синхронизации автомобилей и листинга IAAI. from concurrent.futures import ThreadPoolExecutor -import json import logging +from threading import Event, Thread +import time +import uuid from celery import shared_task from redis import Redis @@ -17,9 +19,7 @@ SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing" def _run_browser_job(func, *args, **kwargs): - # Playwright Sync API нельзя запускать в потоке с активным asyncio loop. - # Celery/зависимости могут поднимать loop в worker-процессе, поэтому - # браузерный код выполняем в отдельном thread без loop. + # Браузерный код запускаем в отдельном потоке без активного loop. with ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser") as executor: future = executor.submit(func, *args, **kwargs) return future.result() @@ -27,7 +27,7 @@ def _run_browser_job(func, *args, **kwargs): def _sync_listing_lock_ttl_seconds() -> int: settings = Settings() - # Небольшой запас к hard time limit задачи, чтобы lock самоснимался после сбоев. + # Небольшой запас к лимиту времени, чтобы lock снимался после сбоев. return max(settings.celery.task_time_limit + 120, 300) @@ -40,35 +40,70 @@ def _get_redis() -> Redis: return Redis.from_url(settings.redis.url, decode_responses=True) -def _release_lock_if_owner(redis_client: Redis, key: str, owner: str) -> None: +def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: try: - current_owner = redis_client.get(key) - if current_owner == owner: - redis_client.delete(key) + return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) except Exception as exc: - logger.warning("Failed to release lock %s: %s", key, exc) - - -def _has_other_active_sync_listing_task(task) -> bool: - try: - inspector = task.app.control.inspect(timeout=1.0) - active_map = inspector.active() or {} - except Exception as exc: - logger.warning("Failed to inspect active tasks: %s", exc) + logger.warning("Failed to acquire lock %s", key, exc_info=True) return False - current_task_id = task.request.id - for worker_tasks in active_map.values(): - for item in worker_tasks or []: - name = str(item.get("name") or "") - task_id = str(item.get("id") or "") - if ( - name == "iaai_scraper.worker.tasks.sync_listing_task" - and task_id - and task_id != current_task_id - ): - return True - return False + +def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: + try: + refreshed = redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) + end + return 0 + """, + 1, + key, + owner_token, + int(ttl_seconds), + ) + return bool(refreshed) + except Exception as exc: + logger.warning("Failed to refresh lock %s", key, exc_info=True) + return None + + +def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: + try: + redis_client.eval( + """ + if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) + end + return 0 + """, + 1, + key, + owner_token, + ) + except Exception as exc: + logger.warning("Failed to release lock %s", key, exc_info=True) + + +def _start_lock_heartbeat( + redis_client: Redis, + key: str, + owner_token: str, + ttl_seconds: int, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) + + def _heartbeat() -> None: + while not stop_event.wait(interval_seconds): + refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) + if refreshed is False: + logger.warning("Lost sync_listing lock ownership for %s", owner_token) + return + + thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) + thread.start() + return stop_event, thread @shared_task( @@ -98,7 +133,7 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"): } except Exception as exc: - logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc) + logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True) raise self.retry(exc=exc) @@ -121,44 +156,15 @@ def sync_listing_task( persistence = _get_persistence() persistence.create_tables() task_id = self.request.id or "unknown" + owner_token = f"{task_id}:{uuid.uuid4().hex}" redis_client = _get_redis() lock_acquired = False lock_ttl = _sync_listing_lock_ttl_seconds() - try: - lock_acquired = bool( - redis_client.set( - SYNC_LISTING_LOCK_KEY, - task_id, - nx=True, - ex=lock_ttl, - ) - ) - except Exception as exc: - logger.warning("Failed to acquire sync lock in Redis: %s", exc) + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None - if not lock_acquired: - # Возможен stale lock после рестарта worker. Если активного sync_listing нет — - # снимаем lock и пытаемся взять его заново. - if not _has_other_active_sync_listing_task(self): - try: - stale_owner = redis_client.get(SYNC_LISTING_LOCK_KEY) - if stale_owner: - logger.warning( - "Removing stale sync lock held by task %s", - stale_owner, - ) - redis_client.delete(SYNC_LISTING_LOCK_KEY) - lock_acquired = bool( - redis_client.set( - SYNC_LISTING_LOCK_KEY, - task_id, - nx=True, - ex=lock_ttl, - ) - ) - except Exception as exc: - logger.warning("Failed to recover stale sync lock: %s", exc) + lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl) if not lock_acquired: logger.info("sync_listing_task skipped: another sync is already running") @@ -169,6 +175,12 @@ def sync_listing_task( } try: + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, + SYNC_LISTING_LOCK_KEY, + owner_token, + lock_ttl, + ) self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) def _job(): with IAAIScraper() as scraper: @@ -198,8 +210,12 @@ def sync_listing_task( return {"status": "success", **summary} except Exception as exc: - logger.error("sync_listing_task failed: %s", exc) + logger.error("sync_listing_task failed: %s", exc, exc_info=True) raise self.retry(exc=exc) finally: + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) if lock_acquired: - _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, task_id) + _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) diff --git a/pyproject.toml b/pyproject.toml index 9f61f9c..7b5357a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "redis>=5.2.0", "sqlalchemy>=2.0.32", "uvicorn>=0.34.0", + "urllib3>=2.0.0", ] [project.optional-dependencies] diff --git a/tests/test_db.py b/tests/test_db.py index 9ee791d..7720590 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -92,16 +92,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase): self.assertEqual(len(images), 1) self.assertIn("imageKeys=2", images[0].fullres_image) - def test_persistence_ignores_non_db_fields(self) -> None: - record = self._record("1000") - - result = self.persistence.upsert_car(record) - - self.assertEqual(result["action"], "inserted") - with self.persistence.session_scope() as session: - car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one() - self.assertEqual(car.origin_id, "1000") - def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: first_run_id = self.persistence.start_sync_run("lane-a") second_run_id = self.persistence.start_sync_run("lane-b") diff --git a/tests/test_listing.py b/tests/test_listing.py index c3d16d0..aae31aa 100644 --- a/tests/test_listing.py +++ b/tests/test_listing.py @@ -31,12 +31,6 @@ class TestListingUnit(unittest.TestCase): page = _FakePage({}) self.assertFalse(ListingCollector._has_next_page(page)) - def test_constructor_with_settings_and_pacer(self) -> None: - settings = Settings() - pacer = HumanPacer(settings) - collector = ListingCollector(settings, pacer) - self.assertIsNotNone(collector) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py index 7c5cd70..9884e17 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -13,15 +13,15 @@ class TestVehicleParserUnit(unittest.TestCase): dom_text = """ Stock #: 45089484 - VIN (Status): - 1HGCM82633A123456 (OK) Primary Damage: Front End + Odometer: + 50,123 mi (Actual) """ result = self.parser._parse_dom_key_value_pairs(dom_text) self.assertEqual(result.get("lot_number"), "45089484") - self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)") self.assertEqual(result.get("primary_damage"), "Front End") + self.assertEqual(result.get("odometer"), "50,123 mi (Actual)") def test_parse_title_for_year_make_model(self) -> None: parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 9fb600f..c053fbf 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -51,15 +51,12 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=2) scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) - scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) - scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) + scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set())) scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]}) - page = MagicMock() - scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")}) - scraper._get_page = MagicMock(return_value=page) - scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [], + }) result = scraper.sync_listing() @@ -67,8 +64,7 @@ class TestScraperSync(unittest.TestCase): self.assertEqual(result["cars_failed"], 0) self.assertIn("trace_id", result) self.assertIn("elapsed_seconds", result) - self.assertEqual(scraper.persistence.upsert_car.call_count, 1) - page.close.assert_called_once() + scraper.sync_batch.assert_called_once() def test_sync_listing_respects_limit(self) -> None: scraper = self._make_scraper() @@ -76,38 +72,25 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=3) scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1}) - scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) - scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) - scraper.collect_listing = MagicMock(return_value={ - "vehicle_urls": [ - "https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/VehicleDetail/333~US", - ] + + # Проверка пути only_new с limit. + scraper._collect_listing_iterative = MagicMock(return_value=( + ["https://www.iaai.com/VehicleDetail/222~US"], + ["https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/VehicleDetail/333~US"], + {"vehicle_urls": [], "pages_collected": 1, "early_stopped": False}, + 0, + )) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [], }) - scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")}) - scraper._get_page = MagicMock(return_value=MagicMock()) scraper.sync_listing(limit=1) - self.assertEqual(scraper._scrape_on_page.call_count, 1) - - def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None: - scraper = self._make_scraper() - - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=4) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0}) - scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set()) - scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set()) - scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]}) - scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")}) - scraper._get_page = MagicMock(return_value=MagicMock()) - - result = scraper.sync_listing() - - self.assertEqual(result["cars_upserted"], 0) + # Должен уйти только один URL. + scraper.sync_batch.assert_called_once() + batch_urls = scraper.sync_batch.call_args[0][0] + self.assertEqual(len(batch_urls), 1) def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None: scraper = self._make_scraper() @@ -115,9 +98,10 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=5) scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) - scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"}) - scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"}) + scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( + {"https://www.iaai.com/VehicleDetail/111~US"}, + {"iaai:222"}, + )) scraper.collect_listing = MagicMock(return_value={ "vehicle_urls": [ @@ -126,26 +110,34 @@ class TestScraperSync(unittest.TestCase): "https://www.iaai.com/VehicleDetail/333~US", # new ] }) - page = MagicMock() - scraper._get_page = MagicMock(return_value=page) - scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")}) + # Возвращаем результат для одного нового авто. + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], + }) result = scraper.sync_listing(only_new=True) self.assertEqual(result["skipped_existing"], 2) self.assertEqual(result["cars_upserted"], 1) - self.assertEqual(scraper._scrape_on_page.call_count, 1) - scraper.persistence.get_existing_origin_urls.assert_called_once() - scraper.persistence.get_existing_origin_ids.assert_called_once() + # В batch должен попасть только новый URL. + scraper.sync_batch.assert_called_once() + batch_urls = scraper.sync_batch.call_args[0][0] + self.assertEqual(len(batch_urls), 1) + self.assertIn("333", batch_urls[0]) + scraper.persistence.get_existing_urls_and_ids.assert_called_once() def test_close_resets_browser_state(self) -> None: scraper = self._make_scraper() + http_pool = MagicMock() + scraper._http_pool = http_pool scraper.context = MagicMock() scraper.browser = MagicMock() scraper.playwright = MagicMock() scraper.close() + http_pool.clear.assert_called_once() + self.assertIsNone(scraper._http_pool) self.assertIsNone(scraper.context) self.assertIsNone(scraper.browser) self.assertIsNone(scraper.playwright) @@ -172,6 +164,11 @@ class TestScraperSync(unittest.TestCase): "https://www.iaai.com/VehicleDetail/999~US", ) + def test_is_protection_or_network_error_detects_known_signals(self) -> None: + self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) + self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) + self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py new file mode 100644 index 0000000..91d778a --- /dev/null +++ b/tests/test_worker_tasks.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock, patch + +from iaai_scraper.worker import tasks + + +class TestWorkerTaskLockHelpers(unittest.TestCase): + def test_acquire_lock_returns_true_on_success(self) -> None: + redis_client = MagicMock() + redis_client.set.return_value = True + + acquired = tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120) + + self.assertTrue(acquired) + redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) + + def test_refresh_lock_if_owner_extends_ttl(self) -> None: + redis_client = MagicMock() + redis_client.eval.return_value = 1 + + refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120) + + self.assertTrue(refreshed) + redis_client.eval.assert_called_once() + + def test_release_lock_if_owner_uses_owner_token(self) -> None: + redis_client = MagicMock() + + tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") + + redis_client.eval.assert_called_once() + args = redis_client.eval.call_args[0] + self.assertEqual(args[1], 1) + self.assertEqual(args[2], "lock:key") + self.assertEqual(args[3], "owner-token") + + def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=False): + persistence = MagicMock() + get_persistence.return_value = persistence + get_redis.return_value = MagicMock() + + tasks.sync_listing_task.push_request(id="task-123") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + persistence.create_tables.assert_called_once() + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "sync_already_running") + + def test_sync_listing_task_releases_owned_lock(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 7, + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 4, + "skipped_existing": 1, + "elapsed_seconds": 1.25, + }): + persistence = MagicMock() + get_persistence.return_value = persistence + redis_client = MagicMock() + get_redis.return_value = redis_client + stop_event = MagicMock() + heartbeat_thread = MagicMock() + start_heartbeat.return_value = (stop_event, heartbeat_thread) + + tasks.sync_listing_task.push_request(id="task-123") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + stop_event.set.assert_called_once() + heartbeat_thread.join.assert_called_once() + release_lock.assert_called_once() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file