diff --git a/.env.example b/.env.example index e7c6f52..02ad08e 100644 --- a/.env.example +++ b/.env.example @@ -55,3 +55,6 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000 CELERY_WORKER_CONCURRENCY=1 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 CELERY_BEAT_SYNC_LIMIT=0 +IAAI_ALWAYS_FULL_SCAN=true +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6 +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600 diff --git a/.gitignore b/.gitignore index 05ffa52..0c27e35 100644 --- a/.gitignore +++ b/.gitignore @@ -19,12 +19,4 @@ build/ artifacts/ celerybeat-schedule* tokens_data/ -tokens.json - -# Local deployment/debug artifacts -/*.tgz -/*.tar.gz -/*.zip -/NOW() -/_speed_check.sql -/.env.bak* \ No newline at end of file +tokens.json \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a6f98e3..c8d8c0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,12 +13,8 @@ RUN pip install --no-cache-dir uv \ && pip install --no-cache-dir -r /tmp/requirements.txt \ && pip install --no-cache-dir playwright-stealth -# Speed-up libs: orjson (fast JSON parse), brotli (HTTP br decompress). -# Pinned outside uv.lock to avoid lock-regen churn. -RUN pip install --no-cache-dir "orjson>=3.10.0" "brotli>=1.1.0" - -# Install both Chromium and Firefox. Chromium is the default engine in Docker -# because it works more reliably with the current IAAI listing page. +# Ставим Chromium и Firefox. +# По умолчанию используем Chromium. RUN python -m playwright install chromium firefox COPY . . @@ -31,6 +27,6 @@ STOPSIGNAL SIGINT USER app -# По умолчанию API, но worker/beat переопределяют CMD в docker-compose +# По умолчанию запускаем API. ENTRYPOINT ["./entrypoint.sh"] CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index c3d93ec..da7ae97 100644 --- a/README.md +++ b/README.md @@ -112,32 +112,7 @@ docker compose up -d `entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд. -По умолчанию `beat` запускает синхронизацию **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`). - -Текущая стратегия устойчива к багам пагинации IAAI: - -- **первый запуск** делает полный bootstrap через `sitemap`; -- если bootstrap не завершился, автоматически продолжается с чекпоинта; -- **после первого полного прохода** каждый час выполняется проход по `sitemap`. - По умолчанию включён режим **rolling refresh**: - - берутся все URL из sitemap, - - новые URL добавляются сразу, - - исчезнувшие URL помечаются как `is_sold=true`, - - уже записанные авто обновляются **батчами по кругу**, а не все разом. - - Это даёт более стабильную нагрузку и снижает риск не уложиться в час. - - Доступные режимы: - - `IAAI_HOURLY_MODE=rolling_refresh` — рекомендуемый продовый режим; - - `IAAI_HOURLY_MODE=full_refresh` — перепарсить все активные авто за один hourly цикл; - - `IAAI_HOURLY_MODE=diff` — только новые авто + sold. - -За счёт этого система: - -- не зависит от page 39 / 70 / 100, -- не уходит в бесконечный цикл пагинации, -- даёт стабильный hourly refresh уже записанных авто без монолитного полного hourly прохода, -- остаётся быстрой и устойчивой при hourly sync. +По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`). Swagger-документация: `http://localhost:8000/docs` @@ -149,9 +124,10 @@ docker compose ps - `worker` имеет healthcheck через `celery inspect ping` - `beat` имеет healthcheck по файлу `celerybeat-schedule` + ## API -**Здоровье и статистика:** +**Статистика:** - `GET /health` — статус сервиса и подключения к БД - `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов @@ -267,11 +243,6 @@ pytest -q ## `pyproject.toml` + `uv.lock` -Проект переведён на современную схему зависимостей: - -- `pyproject.toml` — декларация зависимостей и метаданных проекта -- `uv.lock` — зафиксированные версии для воспроизводимых установок - Локально можно использовать: ```bash @@ -279,10 +250,6 @@ uv sync uv run pytest -q ``` -## Runtime-фильтры - -Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей. - ### Секция `sync` Поддерживаются поля: diff --git a/alembic/versions/004_add_ingestion_tables.py b/alembic/versions/004_add_ingestion_tables.py index df204f0..3ce4466 100644 --- a/alembic/versions/004_add_ingestion_tables.py +++ b/alembic/versions/004_add_ingestion_tables.py @@ -1,8 +1,7 @@ -# Добавление таблиц для новой архитектуры ingestion: candidates, raw snapshots, parse results, retry queue +# Placeholder migration (ingestion tables not yet needed) from typing import Sequence, Union from alembic import op -import sqlalchemy as sa revision: str = "004_add_ingestion_tables" down_revision: Union[str, None] = "003_add_composite_indexes" @@ -11,67 +10,8 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # Таблица vehicle_candidates - op.create_table( - "vehicle_candidates", - sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True), - sa.Column("url", sa.String(), nullable=False, unique=True), - sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.Column("status", sa.String(20), nullable=False, server_default="pending"), - sa.Column("priority", sa.Integer(), nullable=False, server_default="0"), - sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), - ) - - op.create_index("ix_vehicle_candidates_url", "vehicle_candidates", ["url"]) - op.create_index("ix_vehicle_candidates_status_discovered", "vehicle_candidates", ["status", "discovered_at"]) - - # Таблица vehicle_raw_snapshots - op.create_table( - "vehicle_raw_snapshots", - sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True), - sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False), - sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.Column("method", sa.String(20), nullable=False), - sa.Column("success", sa.Boolean(), nullable=False), - sa.Column("raw_data", sa.Text(), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - ) - - op.create_index("ix_vehicle_raw_snapshots_candidate_id", "vehicle_raw_snapshots", ["candidate_id"]) - op.create_index("ix_vehicle_raw_snapshots_captured_at", "vehicle_raw_snapshots", ["captured_at"]) - - # Таблица vehicle_parse_results - op.create_table( - "vehicle_parse_results", - sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True), - sa.Column("snapshot_id", sa.Integer(), sa.ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False), - sa.Column("parsed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), - sa.Column("success", sa.Boolean(), nullable=False), - sa.Column("parsed_data", sa.Text(), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - ) - - op.create_index("ix_vehicle_parse_results_snapshot_id", "vehicle_parse_results", ["snapshot_id"]) - op.create_index("ix_vehicle_parse_results_parsed_at", "vehicle_parse_results", ["parsed_at"]) - - # Таблица vehicle_retry_queue - op.create_table( - "vehicle_retry_queue", - sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True), - sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False), - sa.Column("reason", sa.String(50), nullable=False), - sa.Column("retry_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), - sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"), - ) - - op.create_index("ix_vehicle_retry_queue_candidate_id", "vehicle_retry_queue", ["candidate_id"]) - op.create_index("ix_vehicle_retry_queue_retry_at", "vehicle_retry_queue", ["retry_at"]) + pass def downgrade() -> None: - op.drop_table("vehicle_retry_queue") - op.drop_table("vehicle_parse_results") - op.drop_table("vehicle_raw_snapshots") - op.drop_table("vehicle_candidates") \ No newline at end of file + pass diff --git a/deploy_vps.sh b/deploy_vps.sh deleted file mode 100644 index d9af710..0000000 --- a/deploy_vps.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -APP_DIR="/opt/iaai_scraper_project" -REPO_URL="${1:-}" - -if [[ -z "${REPO_URL}" ]]; then - echo "Usage: ./deploy_vps.sh " - exit 1 -fi - -export DEBIAN_FRONTEND=noninteractive - -apt-get update -apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - git \ - gnupg \ - lsb-release - -install -m 0755 -d /etc/apt/keyrings -if [[ ! -f /etc/apt/keyrings/docker.gpg ]]; then - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg -fi - -chmod a+r /etc/apt/keyrings/docker.gpg -ARCH="$(dpkg --print-architecture)" -CODENAME="$(. /etc/os-release && echo "$VERSION_CODENAME")" -echo \ - "deb [arch=${ARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${CODENAME} stable" \ - > /etc/apt/sources.list.d/docker.list - -apt-get update -apt-get install -y --no-install-recommends \ - docker-ce \ - docker-ce-cli \ - containerd.io \ - docker-buildx-plugin \ - docker-compose-plugin - -mkdir -p /opt - -if [[ -d "${APP_DIR}/.git" ]]; then - git -C "${APP_DIR}" fetch --all --prune - git -C "${APP_DIR}" reset --hard origin/HEAD -else - rm -rf "${APP_DIR}" - git clone "${REPO_URL}" "${APP_DIR}" -fi - -cd "${APP_DIR}" - -if [[ ! -f .env ]]; then - cp .env.vps.example .env -fi - -docker compose down --remove-orphans || true -docker compose up -d --build postgres redis migrate api worker beat -docker compose ps diff --git a/docker-compose.yml b/docker-compose.yml index b49cb53..72d0a19 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ x-app-env: &app-env - # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here + # Всегда используем Postgres. IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} @@ -8,20 +8,28 @@ x-app-env: &app-env CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} - # 0 => без лимита (в коде интерпретируется как None). - # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто. + # 0 = без лимита. CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} - CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3} + CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} - IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40} + IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3} + IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9} + IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8} IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true} IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto} IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999} IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000} + IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true} IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true} IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json} IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} IAAI_BROWSER_ENGINE: chromium + # Self-heal для worker. + IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true} + IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} + IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900} + IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} + IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} TZ: ${TZ:-UTC} x-env-file: &env-file @@ -42,7 +50,7 @@ x-worker-service: &worker-service - tokens_data:/data services: - # PostgreSQL + # PostgreSQL. postgres: image: postgres:16-alpine container_name: iaai-postgres @@ -66,7 +74,7 @@ services: max-size: "10m" max-file: "5" - # Redis (Celery broker) + # Redis. redis: image: redis:7-alpine container_name: iaai-redis @@ -90,7 +98,7 @@ services: max-size: "10m" max-file: "5" - # DB migrations + # Миграции. migrate: <<: *app-service container_name: iaai-migrate @@ -100,7 +108,7 @@ services: condition: service_healthy command: alembic upgrade head - # FastAPI + # API. api: <<: *app-service container_name: iaai-api @@ -128,14 +136,11 @@ services: max-size: "10m" max-file: "5" - # Celery Worker + # Worker. worker: <<: *worker-service - container_name: iaai-worker restart: unless-stopped init: true - mem_limit: ${IAAI_WORKER_MEM_LIMIT:-2g} - memswap_limit: ${IAAI_WORKER_MEM_LIMIT:-2g} depends_on: migrate: condition: service_completed_successfully @@ -146,9 +151,10 @@ services: celery -A iaai_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --pidfile=/tmp/celery-worker.pid - -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-1} + -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} healthcheck: - test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"] + # Проверка worker. + test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"] interval: 60s timeout: 10s retries: 3 @@ -159,7 +165,7 @@ services: max-size: "10m" max-file: "5" - # Celery Beat (периодический планировщик) + # Beat. beat: <<: *worker-service container_name: iaai-beat diff --git a/entrypoint.sh b/entrypoint.sh index 078cf3f..ac236dc 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -55,6 +55,30 @@ start_proxy_bridge_if_needed() { echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})" } +start_self_heal_watchdog_if_worker() { + if ! is_worker_command "$@"; then + return 0 + fi + + if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then + echo "[entrypoint] Self-heal watchdog disabled" + return 0 + fi + + echo "[entrypoint] Starting self-heal watchdog for worker..." + python -m iaai_scraper.worker.self_heal & + SELF_HEAL_PID=$! + sleep 1 + + if ! kill -0 "${SELF_HEAL_PID}" 2>/dev/null; then + echo "[entrypoint] ERROR: self-heal watchdog failed to start" + exit 1 + fi + + echo "[entrypoint] Self-heal watchdog started (PID ${SELF_HEAL_PID})" +} + start_proxy_bridge_if_needed "$@" +start_self_heal_watchdog_if_worker "$@" exec "$@" diff --git a/iaai_scraper/browser/factory.py b/iaai_scraper/browser/factory.py index eb1f578..ccba6cf 100644 --- a/iaai_scraper/browser/factory.py +++ b/iaai_scraper/browser/factory.py @@ -15,7 +15,7 @@ logger = logging.getLogger("iaai_scraper.browser") def _build_init_script() -> str: - # Патч признаков автоматизации. + # Маскировка браузера. hardware_concurrency = random.choice([4, 8, 12, 16]) device_memory = random.choice([4, 8, 16]) languages = ["en-US", "en"] @@ -69,11 +69,10 @@ class BrowserFactory: self.settings = settings def _resolve_engine(self) -> str: - # Выбор движка браузера. + # Выбор движка. engine = self.settings.browser_engine.strip().lower() if engine == "auto": - # Для IAAI в headless-режиме Chromium со stealth-скриптами - # значительно стабильнее Firefox по anti-bot. + # Для IAAI стабильнее Chromium. return "chromium" if engine in ("firefox", "chromium"): return engine @@ -90,11 +89,11 @@ class BrowserFactory: if proxy_dict: launch_kwargs["proxy"] = proxy_dict logger.info("Using proxy: %s", self.settings.proxy.server) - # Параметры Firefox для headless-режима. + # Настройки Firefox. launch_kwargs["firefox_user_prefs"] = { "dom.webdriver.enabled": False, "useAutomationExtension": False, - # Базовые оптимизации для VPS. + # Базовые оптимизации. "media.autoplay.default": 5, "media.volume_scale": "0.0", "media.audio.playback.standalone": False, @@ -111,7 +110,7 @@ class BrowserFactory: logger.info("Launching Firefox (headless=%s)", self.settings.headless) return playwright.firefox.launch(**launch_kwargs) - # Путь запуска Chromium. + # Запуск Chromium. args = [ "--disable-blink-features=AutomationControlled", "--no-default-browser-check", @@ -154,7 +153,7 @@ class BrowserFactory: } if is_firefox: - # Заголовки и user-agent для Firefox. + # Заголовки Firefox. ctx_kwargs["user_agent"] = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " "Gecko/20100101 Firefox/128.0" @@ -183,7 +182,7 @@ class BrowserFactory: context.set_default_navigation_timeout(self.settings.default_timeout_ms) if not is_firefox: - # Патчи маскировки для Chromium. + # Маскировка Chromium. context.add_init_script(_build_init_script()) if stealth_sync: context.on("page", lambda page: stealth_sync(page)) @@ -193,7 +192,7 @@ class BrowserFactory: @staticmethod def enable_resource_blocking(page) -> None: - # Блокируем тяжёлые ресурсы для ускорения загрузки страниц. + # Блокируем тяжёлые ресурсы. BLOCKED_TYPES = {"image", "stylesheet", "font", "media"} BLOCKED_URL_PATTERNS = ( "google-analytics", "googletagmanager", "facebook.net", diff --git a/iaai_scraper/browser/listing.py b/iaai_scraper/browser/listing.py index d2f56d7..f9cb949 100644 --- a/iaai_scraper/browser/listing.py +++ b/iaai_scraper/browser/listing.py @@ -3,12 +3,13 @@ import re import time from dataclasses import asdict, dataclass, field from typing import Any -from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse +from urllib.parse import urljoin from playwright.sync_api import Page from .pace import HumanPacer from ..core.config import Settings +from ..core.utils import first_non_empty logger = logging.getLogger("iaai_scraper.listing") VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) @@ -43,7 +44,6 @@ class ListingPageResult: class ListingCollector: - _DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE = 40 _NEXT_PAGE_SELECTORS: tuple[str, ...] = ( "a[aria-label*='Next']", "button[aria-label*='Next']", @@ -103,33 +103,7 @@ class ListingCollector: return None @staticmethod - def _get_vehicle_link_fingerprint(page: Page, limit: int = 5) -> tuple[str, ...]: - try: - values = page.evaluate( - """ - (limit) => { - return Array.from(document.querySelectorAll("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']")) - .map((el) => (el.getAttribute('href') || '').trim()) - .filter(Boolean) - .slice(0, limit); - } - """, - limit, - ) - if not isinstance(values, list): - return tuple() - return tuple(str(value) for value in values if value) - except Exception: - return tuple() - - @staticmethod - def _wait_for_navigation_result( - page: Page, - old_first_href: str, - expected_page_number: int | None, - *, - old_fingerprint: tuple[str, ...] = (), - ) -> bool: + def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool: if old_first_href: try: page.wait_for_function( @@ -143,12 +117,9 @@ class ListingCollector: except Exception: pass - new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page) if expected_page_number is not None: current_page = ListingCollector._get_current_page_number(page) - if current_page == expected_page_number and ( - not old_fingerprint or (new_fingerprint and new_fingerprint != old_fingerprint) - ): + if current_page == expected_page_number: return True try: @@ -156,81 +127,33 @@ class ListingCollector: except Exception: pass - if not new_fingerprint: - new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page) current_page = ListingCollector._get_current_page_number(page) - if new_fingerprint and old_fingerprint and new_fingerprint != old_fingerprint: - return current_page is None or expected_page_number is None or current_page == expected_page_number if expected_page_number is not None and current_page == expected_page_number: - return not old_fingerprint + return True - return not old_first_href and (not old_fingerprint or bool(new_fingerprint)) - - def _extract_next_page_href(self, page: Page, expected_page_number: int | None = None) -> str | None: - try: - href = page.evaluate( - """ - (expectedPageNumber) => { - const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); - const disabled = (el) => { - if (!el) return true; - const cls = (el.getAttribute('class') || '').toLowerCase(); - const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); - return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); - }; - - const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')) - .filter((el) => visible(el) && !disabled(el)); - - const cleanHref = (el) => { - const href = (el?.getAttribute('href') || '').trim(); - if (!href || href.startsWith('javascript:') || href.startsWith('#')) { - return ''; - } - return href; - }; - - if (expectedPageNumber !== null && expectedPageNumber !== undefined) { - const numeric = controls.find((el) => { - const text = (el.textContent || '').trim(); - return /^\d+$/.test(text) && parseInt(text, 10) === expectedPageNumber; - }); - const numericHref = cleanHref(numeric); - if (numericHref) { - return numericHref; - } - } - - const explicitNext = controls.find((el) => { - const text = (el.textContent || '').trim().toLowerCase(); - const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); - const title = (el.getAttribute('title') || '').trim().toLowerCase(); - const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); - const cls = (el.getAttribute('class') || '').trim().toLowerCase(); - const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); - return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text); - }); - return cleanHref(explicitNext); - } - """, - expected_page_number, - ) - if not href: - return None - return urljoin(page.url or self.settings.home_url, str(href)) - except Exception: - return None + return not old_first_href @staticmethod - def _build_listing_page_url(url: str, page_number: int) -> str: - parsed = urlparse(url) - query_items = [ - (key, value) - for key, value in parse_qsl(parsed.query, keep_blank_values=True) - if key.lower() not in {"page", "pagenumber", "currentpage"} - ] - query_items.append(("page", str(page_number))) - return urlunparse(parsed._replace(query=urlencode(query_items))) + def has_page_number(page: Page, target_page_number: int) -> bool: + try: + return bool(page.evaluate( + """ + (targetPageNumber) => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div')); + return controls.some((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber; + }); + } + """, + target_page_number, + )) + except Exception: + return False def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None: url = url_override or self.settings.listing.cars_url @@ -301,31 +224,6 @@ class ListingCollector: # Короткая пауза вместо длинного sleep. time.sleep(1.0) - def _get_listing_html(self, page: Page, page_number: int) -> str: - try: - return page.locator("html").inner_html(timeout=5_000) - except Exception as exc: - logger.warning("listing html read failed on page %d: %s", page_number, exc) - return "" - - def _has_next_page_from_html(self, html: str, current_page_number: int | None = None) -> bool: - if not html: - return False - normalized = html.replace("\\/", "/") - if re.search( - r"rel\s*=\s*['\"]next['\"]|aria-label\s*=\s*['\"][^'\"]*next|title\s*=\s*['\"][^'\"]*next|class\s*=\s*['\"][^'\"]*next|icon-arrow-right|arrow-right|>\s*next\s*<|>\s*[›»>]\s*<", - normalized, - re.IGNORECASE, - ): - return True - if current_page_number is not None: - next_page = current_page_number + 1 - if re.search(rf">\s*{next_page}\s*<", normalized, re.IGNORECASE): - return True - if re.search(rf"page={next_page}(?:\D|$)", normalized, re.IGNORECASE): - return True - return False - def apply_filters( self, page: Page, @@ -403,30 +301,74 @@ class ListingCollector: return False def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: - # Для IAAI HTML/hydration-извлечение стабильнее, чем прямой DOM eval. + # Считываем ссылки одним проходом по DOM. self._accept_cookie_banner(page) self._wait_for_listing_content(page) + try: + raw_items = page.eval_on_selector_all( + "a[href], [data-href], [href]", + """ + (nodes) => nodes.map((node) => ({ + href: + node.getAttribute('href') || + node.getAttribute('data-href') || + node.getAttribute('data-url') || + '', + title: node.getAttribute('title') || node.getAttribute('aria-label') || '', + text: (node.textContent || '').trim(), + })) + """, + ) + except Exception as exc: + logger.warning("collect_current_page failed on page %d: %s", page_number, exc) + raw_items = [] + total = min(len(raw_items), self.settings.listing.page_link_limit) links: list[ListingVehicleLink] = [] seen: set[str] = set() - html = self._get_listing_html(page, page_number) - for absolute, lot_number in self._extract_vehicle_links_from_html(html): + for idx in range(total): + item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} + href = str(item.get("href") or "") + match = VEHICLE_HREF_RE.search(href) + if not match: + continue + lot_number = match.group(1) + absolute = urljoin(self.settings.home_url, match.group(0)) if absolute in seen: continue seen.add(absolute) - links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) + title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" + links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) if len(links) >= self.settings.listing.max_vehicles_per_run: break - if links: - logger.info( - "Page %d: recovered %d vehicle links from HTML fallback", - page_number, - len(links), - ) + # Fallback: на IAAI ссылки иногда не рендерятся как , + # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). + if not links: + try: + page.wait_for_timeout(1_500) + except Exception: + pass + try: + html = page.content() + except Exception as exc: + logger.debug("page.content() failed on page %d: %s", page_number, exc) + html = "" - next_page_detected = self._has_next_page_from_html(html, current_page_number=page_number) - if not next_page_detected and not html: - next_page_detected = self._has_next_page(page) + for absolute, lot_number in self._extract_vehicle_links_from_html(html): + if absolute in seen: + continue + seen.add(absolute) + links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + if links: + logger.info( + "Page %d: recovered %d vehicle links from HTML fallback", + page_number, + len(links), + ) + next_page_detected = self._has_next_page(page) return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]: @@ -453,59 +395,12 @@ class ListingCollector: def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool: # Запоминаем первую ссылку текущей страницы для определения смены контента. old_first_href = "" - old_fingerprint: tuple[str, ...] = tuple() try: first_link = page.locator(VEHICLE_LINK_SELECTOR).first if first_link.count() > 0: old_first_href = first_link.get_attribute("href") or "" except Exception: pass - old_fingerprint = self._get_vehicle_link_fingerprint(page) - - if expected_page_number is not None and expected_page_number > 1: - direct_page_url = self._build_listing_page_url( - page.url or self.settings.listing.cars_url, - expected_page_number, - ) - try: - logger.debug("Navigating directly to listing page %d via URL: %s", expected_page_number, direct_page_url) - page.goto(direct_page_url, wait_until="domcontentloaded", timeout=15_000) - if self._wait_for_navigation_result( - page, - old_first_href, - expected_page_number, - old_fingerprint=old_fingerprint, - ): - self.pacer.after_page_change() - return True - except Exception as exc: - logger.debug("Direct page-number navigation failed for page %d via %s: %s", expected_page_number, direct_page_url, exc) - - next_href = self._extract_next_page_href(page, expected_page_number) - if next_href: - try: - logger.debug("Navigating directly to next listing page: %s", next_href) - page.goto(next_href, wait_until="domcontentloaded", timeout=15_000) - if self._wait_for_navigation_result( - page, - old_first_href, - expected_page_number, - old_fingerprint=old_fingerprint, - ): - self.pacer.after_page_change() - return True - except Exception as exc: - logger.debug("Direct next-page navigation failed for %s: %s", next_href, exc) - - if ( - expected_page_number is not None - and expected_page_number >= self._DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE - ): - logger.warning( - "Deep pagination direct navigation failed for page %d; skipping flaky UI pagination fallbacks", - expected_page_number, - ) - return False for selector in self._NEXT_PAGE_SELECTORS: locator = page.locator(selector).first @@ -525,12 +420,7 @@ class ListingCollector: except Exception: continue - if self._wait_for_navigation_result( - page, - old_first_href, - expected_page_number, - old_fingerprint=old_fingerprint, - ): + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): self.pacer.after_page_change() return True @@ -590,12 +480,7 @@ class ListingCollector: """ )) if clicked: - if self._wait_for_navigation_result( - page, - old_first_href, - expected_page_number, - old_fingerprint=old_fingerprint, - ): + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): self.pacer.after_page_change() return True except Exception as exc: @@ -626,12 +511,7 @@ class ListingCollector: """ )) if clicked: - if self._wait_for_navigation_result( - page, - old_first_href, - expected_page_number, - old_fingerprint=old_fingerprint, - ): + if self._wait_for_navigation_result(page, old_first_href, expected_page_number): self.pacer.after_page_change() return True except Exception as exc: @@ -716,18 +596,10 @@ class ListingCollector: len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination + or not page_result.next_page_detected ): break - - blind_page_probe = not page_result.next_page_detected - if not self.go_to_next_page(page, expected_page_number=page_number + 1): - if blind_page_probe: - logger.info( - "Stopping pagination on page %d: direct page probe for %d failed and no next-page control was detected", - page_number, - page_number + 1, - ) - break + if not self.go_to_next_page(page): break return { @@ -775,8 +647,22 @@ class ListingCollector: @staticmethod def _has_next_page(page: Page) -> bool: for selector in ListingCollector._NEXT_PAGE_SELECTORS: - if page.locator(selector).count() > 0: - return True + locator = page.locator(selector) + count = locator.count() + if count == 0: + continue + # Проверяем, что хотя бы один элемент не disabled. + # Disabled "Next" на последней странице не означает наличия следующей. + for i in range(min(count, 3)): + try: + el = locator.nth(i) + disabled_attr = el.get_attribute("disabled", timeout=300) + aria_disabled = el.get_attribute("aria-disabled", timeout=300) + cls = (el.get_attribute("class", timeout=300) or "").lower() + if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls: + return True + except Exception: + continue try: return bool(page.evaluate( """ diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py index 295c10f..c3b76d5 100644 --- a/iaai_scraper/cli.py +++ b/iaai_scraper/cli.py @@ -38,17 +38,6 @@ def build_parser() -> argparse.ArgumentParser: sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json")) - # Новые команды для ingestion pipeline - subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap") - - fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates") - fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process") - - enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots") - enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process") - - subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)") - return parser @@ -75,39 +64,6 @@ def main() -> None: data = scraper.scrape_vehicle_detail(args.vehicle_url) elif args.command == "sync-vehicle": data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) - elif args.command == "sync-listing": - only_new = None if args.only_new is None else args.only_new == "true" - data = scraper.sync_listing( - make=args.make, - model=args.model, - lane=args.lane, - limit=args.limit, - only_new=only_new, - ) - elif args.command == "discover-vehicles": - from .discovery_service import DiscoveryService - discovery = DiscoveryService() - count = discovery.discover_new_vehicles() - print(f"Discovered {count} new vehicle candidates") - return - elif args.command == "fetch-pending": - from .fetch_service import FetchService - fetch = FetchService() - count = fetch.process_pending_candidates(limit=args.limit) - print(f"Successfully fetched {count} candidates") - return - elif args.command == "enrich-snapshots": - from .enrichment_service import EnrichmentService - enrichment = EnrichmentService() - count = enrichment.process_unparsed_snapshots(limit=args.limit) - print(f"Successfully enriched {count} snapshots") - return - elif args.command == "run-pipeline": - from .scheduler_service import SchedulerService - scheduler = SchedulerService() - scheduler.run_full_pipeline() - print("Pipeline completed") - return else: only_new = None if args.only_new is None else args.only_new == "true" data = scraper.sync_listing( diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py index f2a087e..859079d 100644 --- a/iaai_scraper/core/config.py +++ b/iaai_scraper/core/config.py @@ -106,7 +106,6 @@ class ListingConfig: cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999) max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000) - resume_max_nav_pages: int = _env_int("IAAI_LISTING_RESUME_MAX_NAV_PAGES", 90) page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500) include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True) collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False) @@ -119,22 +118,6 @@ class ListingConfig: listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "") -@dataclass(slots=True) -class DiscoveryConfig: - mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap") - hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh") - hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 3000) - sitemap_timeout_seconds: int = _env_int("IAAI_SITEMAP_TIMEOUT_SECONDS", 30) - sitemap_retry_attempts: int = _env_int("IAAI_SITEMAP_RETRY_ATTEMPTS", 4) - sitemap_retry_backoff_seconds: float = _env_float("IAAI_SITEMAP_RETRY_BACKOFF_SECONDS", 1.25) - sitemap_direct_probe_limit: int = _env_int("IAAI_SITEMAP_DIRECT_PROBE_LIMIT", 12) - sitemap_direct_probe_stop_after_misses: int = _env_int( - "IAAI_SITEMAP_DIRECT_PROBE_STOP_AFTER_MISSES", - 3, - ) - sitemap_use_curl_cffi: bool = _env_bool("IAAI_SITEMAP_USE_CURL_CFFI", True) - - # Список брендов IAAI для автоматической сегментации. # Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким. IAAI_DEFAULT_MAKES: tuple[str, ...] = ( @@ -226,6 +209,16 @@ class RedisConfig: health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) +# --- Конфиг Discovery (режим обнаружения, hourly batch) --- + +@dataclass(slots=True) +class DiscoveryConfig: + mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap") + hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh") + hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500) + always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", True) + + # --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- @dataclass(slots=True) @@ -242,7 +235,7 @@ class CeleryConfig: beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8) - parallel_segments: bool = _env_bool("IAAI_PARALLEL_SEGMENTS", False) + parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False) block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True) @@ -297,11 +290,11 @@ class Settings: capture: CaptureConfig = field(default_factory=CaptureConfig) pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) listing: ListingConfig = field(default_factory=ListingConfig) - discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) database: DatabaseConfig = field(default_factory=DatabaseConfig) redis: RedisConfig = field(default_factory=RedisConfig) celery: CeleryConfig = field(default_factory=CeleryConfig) proxy: ProxyConfig = field(default_factory=ProxyConfig) + discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) @property def parallel_tabs(self) -> int: diff --git a/iaai_scraper/discovery/__init__.py b/iaai_scraper/discovery/__init__.py index c7d7e49..86402b2 100644 --- a/iaai_scraper/discovery/__init__.py +++ b/iaai_scraper/discovery/__init__.py @@ -1,17 +1,15 @@ from .sitemap import ( - DEFAULT_SITEMAP_INDEX_URL, SitemapDiscoveryError, - SitemapBlockedError, SitemapDiscoveryResult, + SitemapDiscoveryStats, discover_vehicle_urls_from_sitemap, discover_vehicle_urls_from_sitemap_with_stats, ) __all__ = [ - "DEFAULT_SITEMAP_INDEX_URL", - "SitemapBlockedError", "SitemapDiscoveryError", "SitemapDiscoveryResult", + "SitemapDiscoveryStats", "discover_vehicle_urls_from_sitemap", "discover_vehicle_urls_from_sitemap_with_stats", ] diff --git a/iaai_scraper/discovery/sitemap.py b/iaai_scraper/discovery/sitemap.py index 9471868..dc50104 100644 --- a/iaai_scraper/discovery/sitemap.py +++ b/iaai_scraper/discovery/sitemap.py @@ -3,222 +3,27 @@ from __future__ import annotations import gzip import io import logging -import random import re -import time -import xml.etree.ElementTree as ET -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Iterable -from urllib.error import HTTPError, URLError -from urllib.parse import urljoin, urlsplit, urlunsplit -from urllib.request import Request, urlopen - -from ..core.config import Settings - -try: - from curl_cffi import requests as curl_requests -except ImportError: # pragma: no cover - optional runtime dependency guard - curl_requests = None +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen, ProxyHandler, build_opener +import xml.etree.ElementTree as ET logger = logging.getLogger("iaai_scraper.discovery.sitemap") DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" +_SITEMAP_TIMEOUT_SECONDS = 30 _LOC_TAG_RE = re.compile(rb"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) -_XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL) -_BLOCK_MARKERS = ( - b"pardon our interruption", - b"incapsula", - b"please stand by", - b"_incapsula_resource", - b"as you were browsing something about your browser", -) -_HTML_MARKERS = (b" None: - self.settings = settings - self.discovery = settings.discovery - self.proxy_url = settings.proxy.server - self.proxy_auth = None - if settings.proxy.username: - password = settings.proxy.password or "" - self.proxy_auth = (settings.proxy.username, password) - self._transport_name = self._resolve_transport_name() - - @property - def transport_name(self) -> str: - return self._transport_name - - def _resolve_transport_name(self) -> str: - if self.discovery.sitemap_use_curl_cffi and curl_requests is not None: - return "curl_cffi" - return "urllib" - - def fetch(self, url: str) -> SitemapFetchResult: - attempts = max(1, int(self.discovery.sitemap_retry_attempts)) - last_exc: Exception | None = None - for attempt in range(1, attempts + 1): - try: - result = self._fetch_once(url) - if result.blocked: - raise SitemapBlockedError(f"Anti-bot page returned for {url}") - return result - except SitemapBlockedError as exc: - last_exc = exc - logger.warning( - "Sitemap fetch blocked (attempt %d/%d, transport=%s): %s", - attempt, - attempts, - self.transport_name, - url, - ) - except Exception as exc: - last_exc = exc - logger.warning( - "Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s", - attempt, - attempts, - self.transport_name, - url, - exc, - ) - if attempt >= attempts: - break - time.sleep(self._backoff_delay(attempt)) - if last_exc is None: - raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}") - if isinstance(last_exc, SitemapDiscoveryError): - raise last_exc - raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc - - def _fetch_once(self, url: str) -> SitemapFetchResult: - if self.transport_name == "curl_cffi": - return self._fetch_with_curl_cffi(url) - return self._fetch_with_urllib(url) - - def _build_headers(self) -> dict[str, str]: - headers = dict(_DEFAULT_HEADERS) - headers["User-Agent"] = self.settings.fingerprint.user_agent - return headers - - def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult: - assert curl_requests is not None - response = curl_requests.get( - url, - headers=self._build_headers(), - timeout=self.discovery.sitemap_timeout_seconds, - impersonate=random.choice(_CURL_IMPERSONATE_CHOICES), - proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None, - proxy_auth=self.proxy_auth, - allow_redirects=True, - ) - payload = self._decode_payload( - payload=response.content, - encoding=(response.headers.get("Content-Encoding") or ""), - url=url, - ) - blocked = _looks_like_block_page(payload) - return SitemapFetchResult( - url=url, - payload=payload, - source="curl_cffi", - blocked=blocked, - status_code=int(response.status_code), - content_type=response.headers.get("Content-Type"), - ) - - def _fetch_with_urllib(self, url: str) -> SitemapFetchResult: - request = Request(url, headers=self._build_headers()) - try: - with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response: - payload = response.read() - decoded = self._decode_payload( - payload=payload, - encoding=(response.headers.get("Content-Encoding") or ""), - url=url, - ) - return SitemapFetchResult( - url=url, - payload=decoded, - source="urllib", - blocked=_looks_like_block_page(decoded), - status_code=getattr(response, "status", None), - content_type=response.headers.get("Content-Type"), - ) - except HTTPError as exc: - payload = exc.read() if hasattr(exc, "read") else b"" - decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url) - blocked = exc.code in {403, 429} or _looks_like_block_page(decoded) - if blocked: - raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc - raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc - except URLError as exc: - raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc - - @staticmethod - def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes: - normalized = (encoding or "").lower().strip() - if normalized == "gzip" or url.lower().endswith(".gz"): - try: - return gzip.GzipFile(fileobj=io.BytesIO(payload)).read() - except OSError: - return payload - return payload - - def _backoff_delay(self, attempt: int) -> float: - base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds)) - jitter = random.uniform(0.05, 0.35) - return base * (2 ** (attempt - 1)) + jitter - - def _is_vehicle_sitemap_url(url: str) -> bool: lowered = url.strip().lower() + # Берём только sitemap с авто. if "sitemapbranches" in lowered or "sitemapauctions" in lowered: return False return lowered.endswith(".xml") or lowered.endswith(".xml.gz") @@ -229,21 +34,38 @@ def _normalize_vehicle_url(url: str) -> str: return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) +def _download_bytes(url: str, proxy_url: str | None = None) -> bytes: + request = Request( + url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36" + ), + "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8", + "Accept-Encoding": "gzip", + }, + ) + if proxy_url: + handler = ProxyHandler({"http": proxy_url, "https": proxy_url}) + opener = build_opener(handler) + response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + else: + response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS) + with response: + payload = response.read() + encoding = str(response.headers.get("Content-Encoding") or "").lower() + if encoding == "gzip" or url.lower().endswith(".gz"): + return gzip.GzipFile(fileobj=io.BytesIO(payload)).read() + return payload + + def _local_name(tag: str) -> str: if "}" in tag: return tag.rsplit("}", 1)[1] return tag -def _looks_like_block_page(payload: bytes) -> bool: - sample = payload[:8192].lower() - if any(marker in sample for marker in _BLOCK_MARKERS): - return True - if any(marker in sample for marker in _HTML_MARKERS) and b"" not in sample: - return True - return False - - def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]: for match in _LOC_TAG_RE.finditer(xml_bytes): try: @@ -255,13 +77,6 @@ def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]: def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]: - if _looks_like_block_page(xml_bytes): - raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML") - if not _XML_PREFIX_RE.search(xml_bytes[:256]): - fallback_values = list(_iter_loc_values_fallback(xml_bytes)) - if fallback_values: - yield from fallback_values - return try: root = ET.fromstring(xml_bytes) except ET.ParseError as exc: @@ -303,139 +118,93 @@ def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool: return any("/vehicledetail/" in url.lower() for url in urls) -def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]: - base_dir = index_url.rsplit("/", 1)[0] + "/" - return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)] - - -def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]: +def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]: logger.info("Downloading sitemap index: %s", index_url) - try: - index_result = downloader.fetch(index_url) - sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)] - if sitemap_urls: - return sitemap_urls - except SitemapBlockedError as exc: - stats.blocked_sitemaps += 1 - logger.warning("Sitemap index blocked, switching to direct probe: %s", exc) - except SitemapDiscoveryError as exc: - stats.malformed_sitemaps += 1 - logger.warning("Sitemap index unusable, switching to direct probe: %s", exc) - - logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe") - return _probe_direct_sitemap_urls(index_url, downloader, stats) - - -def _probe_direct_sitemap_urls( - index_url: str, - downloader: _SitemapDownloader, - stats: SitemapDiscoveryStats, -) -> list[str]: - discovered: list[str] = [] - consecutive_misses = 0 - probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit) - - for sitemap_url in probe_urls: - try: - result = downloader.fetch(sitemap_url) - raw_urls = list(_iter_loc_values(result.payload)) - except SitemapBlockedError: - stats.blocked_sitemaps += 1 - consecutive_misses += 1 - stats.direct_probe_misses += 1 - if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses: - break - continue - except SitemapDiscoveryError: - consecutive_misses += 1 - stats.direct_probe_misses += 1 - if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses: - break - continue - - if not raw_urls: - consecutive_misses += 1 - stats.direct_probe_misses += 1 - if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses: - break - continue - - consecutive_misses = 0 - stats.direct_probe_hits += 1 - discovered.append(sitemap_url) - - return discovered - - -def _collect_vehicle_urls_from_sitemap( - sitemap_url: str, - downloader: _SitemapDownloader, - stats: SitemapDiscoveryStats, -) -> list[str]: - logger.info("Downloading sitemap: %s", sitemap_url) - try: - result = downloader.fetch(sitemap_url) - raw_urls = list(_iter_loc_values(result.payload)) - except SitemapBlockedError as exc: - stats.blocked_sitemaps += 1 - logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc) - return [] - except SitemapDiscoveryError as exc: - stats.malformed_sitemaps += 1 - logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) - return [] - - stats.fetched_sitemaps += 1 - vehicle_urls = _filter_vehicle_urls(raw_urls) - if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): - logger.info("Skipping non-vehicle sitemap %s", sitemap_url) - return [] - - logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) - return vehicle_urls - - -def discover_vehicle_urls_from_sitemap( - index_url: str = DEFAULT_SITEMAP_INDEX_URL, - *, - settings: Settings | None = None, -) -> list[str]: - result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings) - return result.vehicle_urls - - - -def discover_vehicle_urls_from_sitemap_with_stats( - index_url: str = DEFAULT_SITEMAP_INDEX_URL, - *, - settings: Settings | None = None, -) -> SitemapDiscoveryResult: - runtime_settings = settings or Settings() - downloader = _SitemapDownloader(runtime_settings) - stats = SitemapDiscoveryStats(transport=downloader.transport_name) - - sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] if not sitemap_urls: - raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs") + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") all_vehicle_urls: list[str] = [] for sitemap_url in sitemap_urls: - all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats)) + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) deduped = _filter_vehicle_urls(all_vehicle_urls) if not deduped: - raise SitemapDiscoveryError( - "No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue" - ) + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) + return deduped - logger.info( - "Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)", - len(deduped), - stats.transport, - stats.fetched_sitemaps, - stats.blocked_sitemaps, - stats.malformed_sitemaps, - stats.direct_probe_hits, - stats.direct_probe_misses, - ) + +@dataclass +class SitemapDiscoveryStats: + transport: str = "http" + fetched_sitemaps: int = 0 + blocked_sitemaps: int = 0 + malformed_sitemaps: int = 0 + direct_probe_hits: int = 0 + direct_probe_misses: int = 0 + + +@dataclass +class SitemapDiscoveryResult: + vehicle_urls: list[str] = field(default_factory=list) + stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats) + + +def discover_vehicle_urls_from_sitemap_with_stats( + settings=None, + index_url: str = DEFAULT_SITEMAP_INDEX_URL, +) -> SitemapDiscoveryResult: + """Возвращает URL и статистику.""" + proxy_url = None + if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server: + proxy_url = settings.proxy.server + stats = SitemapDiscoveryStats(transport="http") + logger.info("Downloading sitemap index: %s", index_url) + index_xml = _download_bytes(index_url, proxy_url=proxy_url) + sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)] + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url) + raw_urls = list(_iter_loc_values(sitemap_xml)) + stats.fetched_sitemaps += 1 + except SitemapDiscoveryError as exc: + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + stats.malformed_sitemaps += 1 + continue + except Exception as exc: + logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc) + stats.blocked_sitemaps += 1 + continue + + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + continue + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + all_vehicle_urls.extend(vehicle_urls) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps") + logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped)) return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats) diff --git a/iaai_scraper/discovery_service.py b/iaai_scraper/discovery_service.py deleted file mode 100644 index bb91c4b..0000000 --- a/iaai_scraper/discovery_service.py +++ /dev/null @@ -1,129 +0,0 @@ -import logging -from datetime import datetime, timezone -from typing import List - -from sqlalchemy.orm import Session - -from .core.config import settings -from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError -from .storage.db import get_db_session -from .storage.models import VehicleCandidate - -logger = logging.getLogger("iaai_scraper.discovery_service") - - -class DiscoveryService: - """Сервис для обнаружения новых автомобилей через sitemap IAAI.""" - - def __init__(self): - self.db: Session = get_db_session() - - def discover_new_vehicles(self, max_urls: int = None) -> int: - """ - Обнаруживает новые URL автомобилей и добавляет их в vehicle_candidates. - - Args: - max_urls: Максимальное количество URL для обработки (None = все) - - Returns: - Количество добавленных кандидатов - """ - try: - logger.info("Starting vehicle discovery from sitemap") - result = discover_vehicle_urls_from_sitemap_with_stats() - - if not result.urls: - logger.info("No new URLs discovered") - return 0 - - # Ограничиваем количество URL - urls_to_process = result.urls[:max_urls] if max_urls else result.urls - logger.info(f"Discovered {len(urls_to_process)} potential vehicle URLs (from {len(result.urls)} total)") - - # Фильтруем уже существующие кандидаты - existing_urls = self._get_existing_candidate_urls(urls_to_process) - new_urls = [url for url in urls_to_process if url not in existing_urls] - - if not new_urls: - logger.info("All discovered URLs already exist as candidates") - return 0 - - # Добавляем новых кандидатов - added_count = self._add_candidates(new_urls) - logger.info(f"Added {added_count} new vehicle candidates") - - return added_count - - except SitemapDiscoveryError as e: - logger.error(f"Sitemap discovery failed: {e}") - raise - except Exception as e: - logger.error(f"Unexpected error during discovery: {e}") - raise - - def _get_existing_candidate_urls(self, urls: List[str]) -> set: - """Получает множество уже существующих URL кандидатов.""" - if not urls: - return set() - - # Разбиваем на батчи для эффективности - batch_size = 1000 - existing = set() - - for i in range(0, len(urls), batch_size): - batch = urls[i:i + batch_size] - result = self.db.query(VehicleCandidate.url).filter( - VehicleCandidate.url.in_(batch) - ).all() - existing.update(row[0] for row in result) - - return existing - - def _add_candidates(self, urls: List[str]) -> int: - """Добавляет новые кандидаты в базу данных.""" - now = datetime.now(timezone.utc) - candidates = [ - VehicleCandidate( - url=url, - discovered_at=now, - status="pending", - priority=0 - ) - for url in urls - ] - - self.db.add_all(candidates) - self.db.commit() - - return len(candidates) - - def get_pending_candidates(self, limit: int = 100) -> List[VehicleCandidate]: - """Получает кандидатов в статусе 'pending' для обработки.""" - return self.db.query(VehicleCandidate).filter( - VehicleCandidate.status == "pending" - ).order_by(VehicleCandidate.priority.desc(), VehicleCandidate.discovered_at).limit(limit).all() - - def mark_candidate_processing(self, candidate_id: int): - """Помечает кандидата как обрабатываемого.""" - self.db.query(VehicleCandidate).filter( - VehicleCandidate.id == candidate_id - ).update({ - "status": "processing", - "last_attempt_at": datetime.now(timezone.utc), - "attempts": VehicleCandidate.attempts + 1 - }) - self.db.commit() - - def mark_candidate_processed(self, candidate_id: int): - """Помечает кандидата как обработанного.""" - self.db.query(VehicleCandidate).filter( - VehicleCandidate.id == candidate_id - ).update({"status": "processed"}) - self.db.commit() - - def mark_candidate_failed(self, candidate_id: int): - """Помечает кандидата как неудачного.""" - self.db.query(VehicleCandidate).filter( - VehicleCandidate.id == candidate_id - ).update({"status": "failed"}) - self.db.commit() \ No newline at end of file diff --git a/iaai_scraper/enrichment_service.py b/iaai_scraper/enrichment_service.py deleted file mode 100644 index fa36391..0000000 --- a/iaai_scraper/enrichment_service.py +++ /dev/null @@ -1,140 +0,0 @@ -import json -import logging -from datetime import datetime, timezone -from typing import Optional - -from sqlalchemy.orm import Session - -from .core.config import settings -from .parsing.mapper import CarMapper -from .parsing.parser import VehicleParser -from .storage.db import get_db_session -from .storage.models import VehicleRawSnapshot, VehicleParseResult, Car -from .storage.schemas import CarRecord - -logger = logging.getLogger("iaai_scraper.enrichment_service") - - -class EnrichmentService: - """Сервис для парсинга сырых данных и обогащения автомобилей.""" - - def __init__(self): - self.db: Session = get_db_session() - self.parser = VehicleParser() - self.mapper = CarMapper() - - def enrich_vehicle(self, snapshot: VehicleRawSnapshot) -> Optional[VehicleParseResult]: - """ - Парсит snapshot и сохраняет результат. - - Args: - snapshot: Raw snapshot для парсинга - - Returns: - VehicleParseResult если парсинг успешен - """ - logger.info(f"Enriching snapshot {snapshot.id} for candidate {snapshot.candidate_id}") - - if not snapshot.success or not snapshot.raw_data: - logger.warning(f"Snapshot {snapshot.id} has no data to parse") - return self._save_parse_result(snapshot.id, False, None, "No raw data available") - - try: - # Парсим данные - parsed_data = self._parse_raw_data(snapshot.raw_data) - if not parsed_data: - return self._save_parse_result(snapshot.id, False, None, "Parsing failed") - - # Маппим в CarRecord - car_record = self._map_to_car_record(parsed_data) - if not car_record: - return self._save_parse_result(snapshot.id, False, None, "Mapping failed") - - # Сохраняем в cars таблицу - self._save_car(car_record) - - # Сохраняем успешный результат парсинга - return self._save_parse_result(snapshot.id, True, json.dumps(parsed_data)) - - except Exception as e: - error_msg = f"Unexpected error during enrichment: {str(e)}" - logger.error(f"Error enriching snapshot {snapshot.id}: {error_msg}") - return self._save_parse_result(snapshot.id, False, None, error_msg) - - def _parse_raw_data(self, raw_data: str) -> Optional[dict]: - """Парсит сырые данные в словарь.""" - try: - # Если это JSON от browser capture - if raw_data.strip().startswith('{'): - data = json.loads(raw_data) - # Извлекаем vehicle_summary из scrape result - return data.get("vehicle_summary", {}) - - # Если HTML, используем VehicleParser - parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url") - return parsed.get("vehicle_summary", {}) - - except json.JSONDecodeError: - # Если не JSON, пробуем как HTML - try: - parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url") - return parsed.get("vehicle_summary", {}) - except Exception: - return None - - def _map_to_car_record(self, parsed_data: dict) -> Optional[CarRecord]: - """Маппит parsed data в CarRecord.""" - try: - # Используем CarMapper - payload_insights = {} # TODO: extract from raw data if needed - return self.mapper.map_to_car_record("dummy_url", parsed_data, payload_insights) - except Exception as e: - logger.error(f"Mapping failed: {e}") - return None - - def _save_car(self, car_record: CarRecord): - """Сохраняет CarRecord в базу данных.""" - # Используем PersistenceService - from .storage.db import PersistenceService - from .core.config import settings - persistence = PersistenceService(settings) - persistence.create_tables() - upsert_result = persistence.upsert_car(car_record) - logger.info(f"Car upserted: {upsert_result}") - - def _save_parse_result(self, snapshot_id: int, success: bool, - parsed_data: Optional[str], error_message: Optional[str] = None) -> VehicleParseResult: - """Сохраняет результат парсинга.""" - result = VehicleParseResult( - snapshot_id=snapshot_id, - success=success, - parsed_data=parsed_data, - error_message=error_message - ) - self.db.add(result) - self.db.commit() - self.db.refresh(result) - return result - - def process_unparsed_snapshots(self, limit: int = 10) -> int: - """ - Обрабатывает snapshots без результатов парсинга. - - Returns: - Количество успешно обогащенных snapshots - """ - # Находим snapshots без parse results - snapshots = self.db.query(VehicleRawSnapshot).outerjoin( - VehicleParseResult, VehicleRawSnapshot.id == VehicleParseResult.snapshot_id - ).filter( - VehicleRawSnapshot.success == True, - VehicleParseResult.id.is_(None) - ).limit(limit).all() - - enriched_count = 0 - for snapshot in snapshots: - result = self.enrich_vehicle(snapshot) - if result and result.success: - enriched_count += 1 - - return enriched_count \ No newline at end of file diff --git a/iaai_scraper/fetch_service.py b/iaai_scraper/fetch_service.py deleted file mode 100644 index 39874ea..0000000 --- a/iaai_scraper/fetch_service.py +++ /dev/null @@ -1,119 +0,0 @@ -import logging -from datetime import datetime, timezone -from typing import Optional - -from sqlalchemy.orm import Session - -from .core.config import settings -from .scraper import IAAIScraper -from .storage.db import get_db_session -from .storage.models import VehicleCandidate, VehicleRawSnapshot - -logger = logging.getLogger("iaai_scraper.fetch_service") - - -class FetchService: - """Сервис для захвата сырых данных автомобилей (HTTP/browser fallback).""" - - def __init__(self): - self.db: Session = get_db_session() - self.scraper = IAAIScraper() - - def fetch_vehicle_data(self, candidate: VehicleCandidate) -> Optional[VehicleRawSnapshot]: - """ - Захватывает данные для кандидата автомобиля. - - Args: - candidate: Кандидат для обработки - - Returns: - VehicleRawSnapshot если захват успешен, None если неудача - """ - logger.info(f"Fetching data for candidate {candidate.id}: {candidate.url}") - - try: - # Сначала пытаемся HTTP fast-path - raw_data = self._try_http_capture(candidate.url) - if raw_data: - return self._save_snapshot(candidate.id, "http", True, raw_data) - - # Если HTTP не сработал, используем browser fallback - logger.info(f"HTTP failed for {candidate.url}, trying browser fallback") - raw_data = self._try_browser_capture(candidate.url) - if raw_data: - return self._save_snapshot(candidate.id, "browser", True, raw_data) - - # Оба метода failed - logger.warning(f"Both HTTP and browser capture failed for {candidate.url}") - self._save_snapshot(candidate.id, "browser", False, None, "Both capture methods failed") - return None - - except Exception as e: - error_msg = f"Unexpected error during capture: {str(e)}" - logger.error(f"Error fetching {candidate.url}: {error_msg}") - self._save_snapshot(candidate.id, "unknown", False, None, error_msg) - return None - - def _try_http_capture(self, url: str) -> Optional[str]: - """Пытается захватить данные через HTTP.""" - try: - # Используем существующий метод из scraper - # Но нам нужно инициализировать scraper с сессией - # Пока заглушка - интегрируем позже - # Для теста вернем None, чтобы использовать browser - return None - except Exception as e: - logger.debug(f"HTTP capture failed for {url}: {e}") - return None - - def _try_browser_capture(self, url: str) -> Optional[str]: - """Пытается захватить данные через browser.""" - try: - # Используем scrape_vehicle_detail из scraper - with IAAIScraper() as scraper: - result = scraper.scrape_vehicle_detail(url) - # Возвращаем HTML или JSON данные - return result.get("raw_html") or json.dumps(result) - except Exception as e: - logger.debug(f"Browser capture failed for {url}: {e}") - return None - - def _save_snapshot(self, candidate_id: int, method: str, success: bool, - raw_data: Optional[str], error_message: Optional[str] = None) -> VehicleRawSnapshot: - """Сохраняет snapshot в базу данных.""" - snapshot = VehicleRawSnapshot( - candidate_id=candidate_id, - method=method, - success=success, - raw_data=raw_data, - error_message=error_message - ) - self.db.add(snapshot) - self.db.commit() - self.db.refresh(snapshot) - return snapshot - - def process_pending_candidates(self, limit: int = 10) -> int: - """ - Обрабатывает ожидающих кандидатов. - - Returns: - Количество успешно обработанных кандидатов - """ - from .discovery_service import DiscoveryService - discovery = DiscoveryService() - - candidates = discovery.get_pending_candidates(limit) - processed_count = 0 - - for candidate in candidates: - discovery.mark_candidate_processing(candidate.id) - - snapshot = self.fetch_vehicle_data(candidate) - if snapshot and snapshot.success: - discovery.mark_candidate_processed(candidate.id) - processed_count += 1 - else: - discovery.mark_candidate_failed(candidate.id) - - return processed_count \ No newline at end of file diff --git a/iaai_scraper/parsing/mapper.py b/iaai_scraper/parsing/mapper.py index 7998e6a..b31309b 100644 --- a/iaai_scraper/parsing/mapper.py +++ b/iaai_scraper/parsing/mapper.py @@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord class CarMapper: - # Преобразование данных IAAI в CarRecord. + # Маппер IAAI в CarRecord. BODY_MAP = { "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", @@ -48,7 +48,7 @@ class CarMapper: NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord: - # Собираем нормализованную DB-модель. + # Собираем DB-модель. vehicle_summary = vehicle_summary or {} payload_insights = payload_insights or {} core = payload_insights.get("vehicle_core", {}) @@ -77,7 +77,7 @@ class CarMapper: ) color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) - # Пытаемся определить привод из строки двигателя. + # Пробуем взять привод из двигателя. if not drive or drive == "NA": engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")])) if engine_text: @@ -144,7 +144,7 @@ class CarMapper: @classmethod def _to_money_int(cls, value: Any) -> int | None: - # Нормализация стоимости из разных форматов. + # Нормализация цены. if value is None: return None if isinstance(value, bool): @@ -168,14 +168,14 @@ class CarMapper: for number in numbers: clean = number.replace(" ", "") if "," in clean and "." in clean: - # Поддержка 1,234.56 и 1.234,56. + # Поддержка двух форматов. if clean.rfind(",") > clean.rfind("."): clean = clean.replace(".", "").replace(",", ".") else: clean = clean.replace(",", "") elif "," in clean: parts = clean.split(",") - # 123,45 -> 123.45, иначе разделитель тысяч. + # Десятичный или тысячный разделитель. if len(parts[-1]) in {1, 2} and len(parts) == 2: clean = clean.replace(",", ".") else: @@ -214,7 +214,7 @@ class CarMapper: @staticmethod def _parse_odometer(value: Any) -> int: - # Разбор пробега из строк IAAI. + # Разбор пробега. if value is None: return 0 text = str(value).strip() @@ -223,7 +223,7 @@ class CarMapper: lowered = text.lower() if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]): return 0 - # Извлекаем число из строкового формата одометра. + # Ищем число. numbers = re.findall(r"[\d,]+", text) for num_str in numbers: clean = num_str.replace(",", "") @@ -294,7 +294,7 @@ class CarMapper: empty_default: str | None, fallback: str | None, ) -> str | None: - # Общий helper для enum-нормализации. + # Общая нормализация enum. text = self._as_str(value).lower() if not text: return empty_default @@ -329,7 +329,7 @@ class CarMapper: return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"]) def _build_images(self, urls: list[Any]) -> list[ImageRecord]: - # Для imageKeys берем самый большой размер. + # Для imageKeys берём самый большой размер. best_by_key: dict[str, str] = {} key_order: list[str] = [] non_keyed: list[str] = [] @@ -375,11 +375,11 @@ class CarMapper: @classmethod def _generate_parser_id(cls, origin_id: str) -> str: - # Стабильный parser_id по origin_id. + # Стабильный parser_id. digest = hashlib.sha256(origin_id.encode()).digest() alphabet = cls._PARSER_ID_ALPHABET base = len(alphabet) - num = int.from_bytes(digest[:17], "big") # 17 байт = 136 бит, хватает на 22 символа + num = int.from_bytes(digest[:17], "big") # Хватает на 22 символа. chars: list[str] = [] for _ in range(22): num, idx = divmod(num, base) @@ -387,7 +387,7 @@ class CarMapper: return "car-" + "".join(chars) def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: - # Формат: iaai:{lot_number} (аналог copart:94323985). + # Формат: iaai:{lot_number}. for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]: text = self._as_str(value) if text: diff --git a/iaai_scraper/parsing/parser.py b/iaai_scraper/parsing/parser.py index 5a3c244..4d3fee8 100644 --- a/iaai_scraper/parsing/parser.py +++ b/iaai_scraper/parsing/parser.py @@ -10,9 +10,9 @@ logger = logging.getLogger("iaai_scraper.parsers") class VehicleParser: - # Парсер данных страницы автомобиля. + # Парсер страницы авто. - # Регулярные выражения для парсинга и детекции защиты. + # Регулярки парсинга и защиты. _BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE) _CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]) _ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"]) @@ -101,11 +101,11 @@ class VehicleParser: max_fields = len(set(self.DOM_LABEL_MAP.values())) for i, line in enumerate(lines): - # Ранний выход, когда уже нашли все поля. + # Ранний выход. if len(result) >= max_fields: break - # Сценарий 1: "метка: значение" в одной строке. + # Метка и значение в одной строке. colon_pos = line.find(":") if colon_pos > 0: label_part = line[:colon_pos].strip().lower() @@ -116,7 +116,7 @@ class VehicleParser: result[field_name] = value_part continue - # Сценарий 2: метка и значение на соседних строках. + # Метка и значение в соседних строках. clean = line.rstrip(":").strip().lower() clean_alt = clean.rstrip("#").strip() matched_label = None @@ -164,7 +164,7 @@ class VehicleParser: page_title = title_match.group(1).strip() title_parsed = self._parse_title_for_year_make_model(page_title, dom_text) - # Один проход по payload для всех полей. + # Один проход по payload. all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP) summary: dict[str, Any] = {"source_url": vehicle_url} @@ -199,7 +199,7 @@ class VehicleParser: p = item.get("payload") if isinstance(p, (dict, list)): payloads.append(p) - # Дополнительный проход по встроенному JSON. + # Доп. проход по JSON. extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP) for field, vals in extra.items(): if not summary.get(field): @@ -207,7 +207,7 @@ class VehicleParser: if v: summary[field] = v - # Передаём image_urls без повторного извлечения. + # Передаём готовые image_urls. image_urls = summary.get("image_urls") or [] return { "vehicle_summary": summary, @@ -325,7 +325,7 @@ class VehicleParser: for script_text in scripts: if "{" not in script_text and "[" not in script_text: continue - # Пропускаем слишком большие минифицированные блоки. + # Пропускаем большие блоки. if len(script_text) > 51_200: continue try: diff --git a/iaai_scraper/scheduler_service.py b/iaai_scraper/scheduler_service.py deleted file mode 100644 index 07159af..0000000 --- a/iaai_scraper/scheduler_service.py +++ /dev/null @@ -1,69 +0,0 @@ -import logging -import time -from datetime import datetime, timezone, timedelta - -from .core.config import settings -from .discovery_service import DiscoveryService -from .fetch_service import FetchService -from .enrichment_service import EnrichmentService - -logger = logging.getLogger("iaai_scraper.scheduler_service") - - -class SchedulerService: - """Сервис для планирования и координации ingestion pipeline.""" - - def __init__(self): - self.discovery = DiscoveryService() - self.fetch = FetchService() - self.enrichment = EnrichmentService() - - def run_full_pipeline(self): - """Запускает полный цикл ingestion: discovery -> fetch -> enrichment.""" - logger.info("Starting full ingestion pipeline") - - try: - # 1. Discovery phase - logger.info("Phase 1: Discovery") - discovered_count = self.discovery.discover_new_vehicles() - logger.info(f"Discovered {discovered_count} new candidates") - - # 2. Fetch phase - logger.info("Phase 2: Fetch") - fetched_count = self.fetch.process_pending_candidates(limit=50) - logger.info(f"Successfully fetched {fetched_count} candidates") - - # 3. Enrichment phase - logger.info("Phase 3: Enrichment") - enriched_count = self.enrichment.process_unparsed_snapshots(limit=50) - logger.info(f"Successfully enriched {enriched_count} snapshots") - - logger.info("Ingestion pipeline completed") - - except Exception as e: - logger.error(f"Pipeline failed: {e}") - raise - - def run_continuous_pipeline(self, interval_minutes: int = 30): - """Запускает непрерывный цикл ingestion с интервалом.""" - logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals") - - while True: - try: - self.run_full_pipeline() - except Exception as e: - logger.error(f"Pipeline iteration failed: {e}") - - logger.info(f"Sleeping for {interval_minutes} minutes") - time.sleep(interval_minutes * 60) - - def run_targeted_enrichment(self): - """Запускает только enrichment для существующих snapshots.""" - logger.info("Running targeted enrichment") - enriched_count = self.enrichment.process_unparsed_snapshots(limit=100) - logger.info(f"Enriched {enriched_count} snapshots") - - def cleanup_old_data(self, days_to_keep: int = 30): - """Очищает старые данные (опционально).""" - # TODO: implement if needed - pass \ No newline at end of file diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py index 69a8977..8f60914 100644 --- a/iaai_scraper/scraper.py +++ b/iaai_scraper/scraper.py @@ -1,13 +1,14 @@ -import gc import json import logging import os +import random import re import signal import time import uuid import html as html_module from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import TimeoutError as FuturesTimeoutError from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable @@ -21,7 +22,6 @@ from playwright.sync_api import BrowserContext, Page, sync_playwright from playwright.sync_api import TimeoutError as PlaywrightTimeoutError from .browser import BrowserFactory, HumanPacer, NetworkCapture -from .discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats from .core.config import Settings, settings, parse_listing_segments from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError from .core.logs import set_trace_id, setup_logging @@ -39,6 +39,63 @@ VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) HTML_TAG_RE = re.compile(r"<[^>]+>") SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) +# Таймауты операций страницы. +_PAGE_COLLECT_TIMEOUT_S = 90 +_PAGE_NEXT_TIMEOUT_S = 60 +# Ротация контекста выключена по умолчанию. +_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("IAAI_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120 +_SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2 + + +class PageOperationTimeoutError(Exception): + """Browser page operation exceeded per-op watchdog timeout.""" + + +class _PageOpWatchdog: + """SIGALRM-based watchdog. + + Работает только в главном потоке процесса (Celery prefork child — это ок). + В других контекстах — no-op. + """ + + def __init__(self, seconds: int, label: str) -> None: + self.seconds = max(1, int(seconds)) + self.label = label + self._old_handler = None + self._active = False + + def _handler(self, signum, frame): # noqa: ARG002 + raise PageOperationTimeoutError( + f"Page operation '{self.label}' exceeded {self.seconds}s watchdog" + ) + + def __enter__(self): + import threading as _threading + if _threading.current_thread() is not _threading.main_thread(): + return self + if not hasattr(signal, "SIGALRM"): + return self + try: + self._old_handler = signal.signal(signal.SIGALRM, self._handler) + signal.alarm(self.seconds) + self._active = True + except (ValueError, OSError): + # signal можно выставлять только из main thread; в остальном пропускаем. + self._active = False + return self + + def __exit__(self, exc_type, exc, tb): + if not self._active: + return False + try: + signal.alarm(0) + if self._old_handler is not None: + signal.signal(signal.SIGALRM, self._old_handler) + except (ValueError, OSError): + pass + return False + class IAAIScraper: @@ -122,14 +179,17 @@ class IAAIScraper: self.car_mapper = CarMapper() self.persistence = PersistenceService(self.settings) self._shutdown_requested = False - self._progress_callback: Callable[[str, dict[str, Any]], None] | None = None + self._progress_callback: Callable[[str, dict], None] | None = None + # HTTP pool: при parallel_tabs=24 и concurrency=4 пик ≈ 96 конкурентных сокетов; + # даём запас до 256, чтобы не упираться в PoolError под всплесками PX/retry. proxy_url = self.settings.proxy.server if proxy_url: _proxy_kwargs: dict = { "num_pools": 4, "maxsize": 64, + "block": False, "retries": False, - "timeout": urllib3.Timeout(connect=5, read=10), + "timeout": urllib3.Timeout(connect=5, read=20), } if self.settings.proxy.username: _proxy_kwargs["proxy_headers"] = urllib3.make_headers( @@ -141,8 +201,9 @@ class IAAIScraper: self._http_pool = urllib3.PoolManager( num_pools=4, maxsize=64, + block=False, retries=False, - timeout=urllib3.Timeout(connect=5, read=10), + timeout=urllib3.Timeout(connect=5, read=20), ) def _new_trace_id(self, prefix: str) -> str: @@ -151,17 +212,22 @@ class IAAIScraper: set_trace_id(trace_id) return trace_id - def set_progress_callback(self, callback: Callable[[str, dict[str, Any]], None] | None) -> None: + def _reload_runtime_config(self) -> None: + """Reload runtime config from file to apply include/exclude changes without restart.""" + try: + self.runtime_config = RuntimeConfig.from_file(self.settings.runtime_config_file) + except Exception: + logger.warning("Failed to reload runtime config, keeping previous values", exc_info=True) + + def set_progress_callback(self, callback: Callable[[str, dict], None]) -> None: self._progress_callback = callback - def _emit_progress(self, stage: str, **payload: Any) -> None: - callback = self._progress_callback - if callback is None: - return - try: - callback(stage, payload) - except Exception: - logger.debug("progress callback failed for stage=%s", stage, exc_info=True) + def _report_progress(self, stage: str, **meta: Any) -> None: + if self._progress_callback is not None: + try: + self._progress_callback(stage, meta) + except Exception: + pass def __enter__(self) -> "IAAIScraper": if self.playwright is None: @@ -202,7 +268,6 @@ class IAAIScraper: pass finally: self._http_pool = None - gc.collect() def _new_context(self) -> BrowserContext: if self.browser is None: @@ -253,133 +318,6 @@ class IAAIScraper: vehicle_urls.append(normalized) return vehicle_urls - def _should_use_sitemap_discovery( - self, - *, - make: str | None, - model: str | None, - listing_url: str | None, - year_min: int | None, - year_max: int | None, - limit: int | None, - effective_only_new: bool, - ) -> bool: - # Полный unfiltered scan всегда должен идти через sitemap. - # Listing-пагинация на IAAI медленная, нестабильная и может зависать - # на resume/recovery даже когда raw HTTP по карточкам работает быстро. - return ( - make is None - and model is None - and listing_url is None - and year_min is None - and year_max is None - and (limit is None or limit <= 0) - and not effective_only_new - ) - - def _sync_listing_via_sitemap( - self, - *, - lane: str, - limit: int | None, - effective_only_new: bool, - ) -> dict[str, Any]: - del effective_only_new # sitemap full-scan path is used only for full discovery. - discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=self.settings) - vehicle_urls = self._dedupe_urls(discovery_result.vehicle_urls) - if limit is not None and limit > 0: - vehicle_urls = vehicle_urls[:limit] - - batch_size = self.settings.celery.batch_size - total = len(vehicle_urls) - cars_upserted = 0 - cars_failed = 0 - images_upserted = 0 - failures: list[dict[str, str]] = [] - - logger.info("Sitemap-based full discovery found %d vehicle URLs", total) - self._emit_progress( - "sitemap_discovery_done", - urls_found=total, - batch_size=batch_size, - transport=discovery_result.stats.transport, - fetched_sitemaps=discovery_result.stats.fetched_sitemaps, - blocked_sitemaps=discovery_result.stats.blocked_sitemaps, - malformed_sitemaps=discovery_result.stats.malformed_sitemaps, - direct_probe_hits=discovery_result.stats.direct_probe_hits, - direct_probe_misses=discovery_result.stats.direct_probe_misses, - ) - - for batch_start in range(0, total, batch_size): - batch_urls = vehicle_urls[batch_start:batch_start + batch_size] - self._emit_progress( - "listing_batch_start", - source="sitemap", - batch_offset=batch_start, - batch_size=len(batch_urls), - pending_urls=max(0, total - batch_start), - batch_first_url=batch_urls[0] if batch_urls else None, - ) - try: - batch_result = self.sync_batch(batch_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - except Exception as batch_exc: - logger.error( - "Sitemap batch %d-%d failed: %s", - batch_start + 1, - batch_start + len(batch_urls), - batch_exc, - ) - cars_failed += len(batch_urls) - failures.append({ - "vehicle_url": f"sitemap_batch_{batch_start}", - "error": str(batch_exc), - }) - self._emit_progress( - "listing_batch_done", - source="sitemap", - batch_offset=batch_start, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - pending_urls=max(0, total - (batch_start + len(batch_urls))), - ) - - return { - "listing": { - "status": "ok", - "listing_url": self.settings.listing.cars_url, - "applied_filters": { - "make": None, - "model": None, - "year_min": None, - "year_max": None, - }, - "pages_collected": 0, - "vehicles_collected": total, - "vehicle_urls": vehicle_urls, - "early_stopped": False, - "truncated_by_time_budget": False, - "pagination_interrupted": False, - "pages": [], - "source": "sitemap", - "transport": discovery_result.stats.transport, - "blocked_sitemaps": discovery_result.stats.blocked_sitemaps, - "malformed_sitemaps": discovery_result.stats.malformed_sitemaps, - "direct_probe_hits": discovery_result.stats.direct_probe_hits, - "direct_probe_misses": discovery_result.stats.direct_probe_misses, - }, - "total": total, - "skipped_existing": 0, - "cars_upserted": cars_upserted, - "cars_failed": cars_failed, - "images_upserted": images_upserted, - "failures": failures, - "all_listing_origin_urls": set(vehicle_urls), - } - def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: url_to_origin_id = { url: self._extract_db_origin_id_from_url(url) @@ -486,16 +424,21 @@ class IAAIScraper: listing_url: str | None = None, year_min: int | None = None, year_max: int | None = None, - max_nav_pages: int | None = None, + max_nav_pages: int = 10, ) -> tuple[Page, dict[str, str | int | None]]: - if max_nav_pages is None: - max_nav_pages = max(1, self.settings.listing.resume_max_nav_pages) # Ограничиваем глубину навигации: если до цели > max_nav_pages кликов — не пытаемся. if target_page_number > max_nav_pages + 1: raise ListingResumeError( f"Cannot resume at page {target_page_number}: " f"exceeds max navigation depth ({max_nav_pages} pages)" ) + + self._report_progress( + "listing_resume_started", + target_page=target_page_number, + max_nav_pages=max_nav_pages, + ) + page, applied_filters = self._open_listing_page( make=make, model=model, @@ -504,11 +447,35 @@ class IAAIScraper: year_max=year_max, ) + self._report_progress( + "listing_resume_opened", + target_page=target_page_number, + ) + for expected_page in range(2, target_page_number + 1): + # Heartbeat для watchdog: при deep-resume (100+ кликов) задача может + # идти несколько минут без batch_upserted, поэтому пульсуем прогресс. + if expected_page == 2 or expected_page == target_page_number or expected_page % 5 == 0: + self._report_progress( + "listing_resume_progress", + current_page=expected_page - 1, + target_page=target_page_number, + ) + if not self.listing_collector.go_to_next_page(page, expected_page_number=expected_page): + self._report_progress( + "listing_resume_failed", + current_page=expected_page - 1, + target_page=target_page_number, + ) page.close() raise ListingResumeError(f"Failed to resume listing at page {target_page_number}") + self._report_progress( + "listing_resume_completed", + current_page=target_page_number, + target_page=target_page_number, + ) logger.warning("Listing resumed at page %d after recovery", target_page_number) return page, applied_filters @@ -580,11 +547,11 @@ class IAAIScraper: total = 0 cars_upserted = 0 cars_failed = 0 + protection_events = 0 images_upserted = 0 failures: list[dict[str, str]] = [] early_stopped = False truncated_by_time_budget = False - pagination_interrupted = False known_origin_ids: set[str] | None = None threshold = self.settings.listing.early_stop_threshold @@ -630,6 +597,7 @@ class IAAIScraper: batch_result = self.sync_batch(batch_urls, lane=lane) cars_upserted += batch_result.get("cars_upserted", 0) cars_failed += batch_result.get("cars_failed", 0) + protection_events += int(batch_result.get("protection_events", 0)) images_upserted += batch_result.get("images_upserted", 0) failures.extend(batch_result.get("failures", [])) except Exception as batch_exc: @@ -643,6 +611,7 @@ class IAAIScraper: batch_result = self.sync_batch(pending_urls, lane=lane) cars_upserted += batch_result.get("cars_upserted", 0) cars_failed += batch_result.get("cars_failed", 0) + protection_events += int(batch_result.get("protection_events", 0)) images_upserted += batch_result.get("images_upserted", 0) failures.extend(batch_result.get("failures", [])) except Exception as batch_exc: @@ -657,12 +626,13 @@ class IAAIScraper: "cars_upserted": cars_upserted, "cars_failed": cars_failed, "images_upserted": images_upserted, + "protection_events": protection_events, "failures": failures, "all_listing_origin_urls": all_listing_origin_urls, } def _process_pending_batch(batch_urls: list[str], batch_start: int) -> bool: - nonlocal cars_upserted, cars_failed, images_upserted, failures + nonlocal cars_upserted, cars_failed, protection_events, images_upserted, failures if not batch_urls: return True @@ -676,8 +646,15 @@ class IAAIScraper: batch_result = self.sync_batch(batch_urls, lane=lane) cars_upserted += batch_result.get("cars_upserted", 0) cars_failed += batch_result.get("cars_failed", 0) + protection_events += int(batch_result.get("protection_events", 0)) images_upserted += batch_result.get("images_upserted", 0) failures.extend(batch_result.get("failures", [])) + self._report_progress( + "batch_upserted", + cars_upserted=cars_upserted, + cars_failed=cars_failed, + total_discovered=total, + ) return True except PlaywrightError as pw_exc: logger.warning( @@ -715,61 +692,90 @@ class IAAIScraper: year_min=year_min, year_max=year_max, ) - self._emit_progress( - "listing_opened", - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) + pages_since_rotation = 0 for page_number in range(1, self.settings.listing.max_pages_per_run + 1): - self._emit_progress( - "listing_collect_start", + # Heartbeat для worker stall-watchdog: при малом числе ссылок на странице + # batch_upserted может долго не вызываться, поэтому пульсуем прогресс + # на каждом шаге пагинации. + self._report_progress( + "listing_page_scan_started", page_number=page_number, + total_discovered=total, pending_urls=len(pending_urls), - total_new=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, ) - - # Защита от crashes при сборе данных страницы - page_result = None - collect_attempts = 0 - max_collect_attempts = 3 - - while collect_attempts < max_collect_attempts and page_result is None: - collect_attempts += 1 + + # Проактивная ротация контекста (опционально, по умолчанию выключена). + # Если включена — требует долистывания до page_number после reopen, + # поэтому max_nav_pages поднят под реальную глубину. + if _CONTEXT_ROTATE_EVERY_PAGES > 0 and pages_since_rotation >= _CONTEXT_ROTATE_EVERY_PAGES: + logger.warning( + "Rotating browser context at page %d (every %d pages) to reset anti-bot state", + page_number, _CONTEXT_ROTATE_EVERY_PAGES, + ) try: + page.close() + except Exception: + pass + page = None + try: + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + pages_since_rotation = 0 + except ListingResumeError as resume_exc: + logger.warning( + "Context rotation could not resume at page %d (%s) — stopping segment", + page_number, resume_exc, + ) + break + + try: + with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number}"): page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - except (PlaywrightError, PlaywrightTimeoutError, Exception) as collect_exc: - logger.warning("Exception during page collection (page %d, attempt %d/%d): %s", - page_number, collect_attempts, max_collect_attempts, collect_exc) - if collect_attempts < max_collect_attempts: - logger.info("Retrying page collection after 5 seconds...") - time.sleep(5) - try: - page.reload(wait_until="domcontentloaded", timeout=30_000) - except Exception as reload_exc: - logger.debug("Page reload failed during collection retry: %s", reload_exc) - else: - logger.error("All collection attempts failed for page %d — stopping pagination", page_number) - pagination_interrupted = True - page = None - break - - if page_result is None: - break - - self._emit_progress( - "listing_collect_done", - page_number=page_number, - links_found=len(page_result.vehicle_links), - next_page_detected=page_result.next_page_detected, - ) + except (PageOperationTimeoutError, PlaywrightError) as op_exc: + logger.warning( + "Page %d collect stalled/failed (%s) — reopening listing and resuming", + page_number, op_exc, + ) + try: + page.close() + except Exception: + pass + page = None + try: + page, _ = self._reopen_listing_and_resume( + target_page_number=page_number, + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + max_nav_pages=300, + ) + pages_since_rotation = 0 + with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number} (after reopen)"): + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + except (ListingResumeError, PageOperationTimeoutError, PlaywrightError) as resume_exc: + logger.warning( + "Cannot recover at page %d (%s) — stopping pagination for this segment", + page_number, resume_exc, + ) + break + pages_info.append({ "page_number": page_result.page_number, "links_found": len(page_result.vehicle_links), }) + pages_since_rotation += 1 page_urls = self._extract_page_urls( page_result, @@ -779,7 +785,6 @@ class IAAIScraper: ) if not page_urls: - self._emit_progress("listing_empty_page_recovery", page_number=page_number) page_result, page_urls = self._recover_empty_listing_page( page, page_number=page_number, @@ -790,7 +795,6 @@ class IAAIScraper: ) if not page_urls and page_number > 1: logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number) - self._emit_progress("listing_resume_start", page_number=page_number) page.close() try: page, _ = self._reopen_listing_and_resume( @@ -800,6 +804,7 @@ class IAAIScraper: listing_url=listing_url, year_min=year_min, year_max=year_max, + max_nav_pages=300, ) page_result = self.listing_collector.collect_current_page(page, page_number=page_number) page_urls = self._extract_page_urls( @@ -813,7 +818,6 @@ class IAAIScraper: "Cannot resume at page %d (%s) — stopping pagination for this segment", page_number, resume_exc, ) - pagination_interrupted = True page = None page_urls = [] if not page_urls: @@ -864,6 +868,17 @@ class IAAIScraper: total, limit if limit is not None else "∞", ) + self._report_progress( + "listing_page_scan_done", + page_number=page_number, + page_links=len(page_urls), + page_new=len(fresh_urls), + page_known=page_skipped, + total_discovered=total, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) # Прогресс каждые 10 страниц на уровне WARNING. if page_number % 10 == 0: @@ -878,52 +893,129 @@ class IAAIScraper: while len(pending_urls) >= batch_size: batch_urls = pending_urls[:batch_size] - self._emit_progress( - "listing_batch_start", + self._report_progress( + "listing_batch_dispatch_started", page_number=page_number, - pending_urls=len(pending_urls), batch_size=len(batch_urls), - batch_first_url=batch_urls[0] if batch_urls else None, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, ) if not _process_pending_batch(batch_urls, cars_upserted + cars_failed): pending_urls = [] break pending_urls = pending_urls[batch_size:] - self._emit_progress( - "listing_batch_done", + self._report_progress( + "listing_batch_dispatch_done", page_number=page_number, pending_urls=len(pending_urls), cars_upserted=cars_upserted, cars_failed=cars_failed, ) + # Пауза между батчами: снижает нагрузку на IAAI и риск бана. + if pending_urls: + time.sleep(random.uniform(0.5, 1.5)) + + # Anti-stall: не держим хвост pending до следующей страницы/конца сегмента. + # Если после scan остались URL меньше batch_size — отправляем их сразу. + if pending_urls: + self._report_progress( + "listing_tail_batch_started", + page_number=page_number, + batch_size=len(pending_urls), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + if not _process_pending_batch(pending_urls, cars_upserted + cars_failed): + pending_urls = [] + break + pending_urls = [] + self._report_progress( + "listing_tail_batch_done", + page_number=page_number, + pending_urls=0, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) if limit is not None and limit > 0 and total >= limit: break if ( self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination + or not page_result.next_page_detected ): break - self._emit_progress("listing_next_page_start", from_page=page_number, to_page=page_number + 1) - - # Обработка навигации к следующей странице с защитой от crashes - navigation_success = False + + if page_number == 1 and not self.listing_collector.has_page_number(page, 2): + logger.info( + "Page 2 not found on page 1 — treating segment as single-page", + ) + self._report_progress( + "listing_single_page_no_page2", + page_number=page_number, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + break + + suspicious_small_segment = ( + total <= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS + and page_number >= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE + ) + self._report_progress( + "listing_next_page_started", + page_number=page_number, + next_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) try: - navigation_success = self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1) - except (PlaywrightError, PlaywrightTimeoutError, Exception) as nav_exc: - logger.warning("Exception during page navigation (page %d → %d): %s", page_number, page_number + 1, nav_exc) - navigation_success = False - - if not navigation_success: - if not page_result.next_page_detected: - logger.info( - "Failed to navigate to page %d with no next-page control detected — treating pagination as exhausted", - page_number + 1, + with _PageOpWatchdog(_PAGE_NEXT_TIMEOUT_S, f"next page {page_number + 1}"): + next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1) + except (PageOperationTimeoutError, PlaywrightError) as nav_exc: + logger.warning( + "go_to_next_page stalled/failed at page %d (%s) — will reopen", + page_number + 1, nav_exc, + ) + next_ok = False + self._report_progress( + "listing_next_page_done", + page_number=page_number, + next_page_number=page_number + 1, + next_ok=bool(next_ok), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + if not next_ok: + if page_number == 1: + logger.warning( + "Page 2 is not reachable from page 1 — treating segment as single-page and moving on", + ) + break + if suspicious_small_segment: + logger.warning( + "Small segment pagination looks exhausted at page %d: total=%d, next page navigation failed — stopping segment", + page_number, + total, ) break logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1) - self._emit_progress("listing_next_page_failed", from_page=page_number, to_page=page_number + 1) - page.close() + self._report_progress( + "listing_reopen_started", + page_number=page_number, + target_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) + try: + page.close() + except Exception: + pass page = None try: page, _ = self._reopen_listing_and_resume( @@ -933,30 +1025,35 @@ class IAAIScraper: listing_url=listing_url, year_min=year_min, year_max=year_max, + max_nav_pages=300, + ) + pages_since_rotation = 0 + self._report_progress( + "listing_reopen_done", + page_number=page_number, + target_page_number=page_number + 1, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, ) except ListingResumeError as resume_exc: logger.warning( "Cannot resume at page %d (%s) — treating pagination as exhausted", page_number + 1, resume_exc, ) - pagination_interrupted = True + self._report_progress( + "listing_reopen_failed", + page_number=page_number, + target_page_number=page_number + 1, + error=str(resume_exc), + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) break - else: - self._emit_progress("listing_next_page_done", from_page=page_number, to_page=page_number + 1) if pending_urls: - self._emit_progress( - "listing_final_batch_start", - pending_urls=len(pending_urls), - batch_first_url=pending_urls[0] if pending_urls else None, - ) _process_pending_batch(pending_urls, cars_upserted + cars_failed) - self._emit_progress( - "listing_final_batch_done", - pending_urls=0, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) logger.warning( "sync_listing final progress: pages=%d, discovered=%d, upserted=%d, failed=%d", @@ -979,7 +1076,6 @@ class IAAIScraper: "vehicle_urls": all_raw_urls, "early_stopped": early_stopped, "truncated_by_time_budget": truncated_by_time_budget, - "pagination_interrupted": pagination_interrupted, "pages": pages_info, }, "total": total, @@ -987,6 +1083,7 @@ class IAAIScraper: "cars_upserted": cars_upserted, "cars_failed": cars_failed, "images_upserted": images_upserted, + "protection_events": protection_events, "failures": failures, "all_listing_origin_urls": all_listing_origin_urls, } @@ -1353,26 +1450,6 @@ class IAAIScraper: ] return "; ".join(pairs) - def _ensure_http_fast_path_session(self) -> None: - if self.context is not None: - try: - if self._cookie_header_for_context(): - return - except Exception: - logger.debug("Failed to inspect existing context cookies", exc_info=True) - - warmup_page: Page | None = None - try: - warmup_page = self._get_page_with_warmup() - except Exception as exc: - logger.warning("HTTP fast-path warmup failed: %s", exc) - finally: - if warmup_page is not None: - try: - warmup_page.close() - except Exception: - pass - def _scrape_via_raw_http( self, vehicle_url: str, @@ -1466,8 +1543,6 @@ class IAAIScraper: _HTTP_MAX_RETRIES = 2 # Повторов на URL до перехода в браузер _HTTP_RETRY_DELAYS = (0.4, 1.0) # Задержки между повторами _FALLBACK_PARALLEL_PAGES = 4 # Параллельных вкладок для fallback - # _FALLBACK_NAV_TIMEOUT_MS теперь берётся из settings.fallback_navigation_timeout_ms - # (см. использование ниже). Хардкод 8000 убран — слишком мало при rate-limit от Imperva. def _browser_fallback_parallel( self, @@ -1490,14 +1565,24 @@ class IAAIScraper: local_failed = 0 local_protection = 0 page: Page | None = None + processed_local = 0 try: for url, global_idx in url_slice: - self._emit_progress( - "browser_fallback_open_start", - vehicle_url=url, + processed_local += 1 + # Heartbeat в Redis progress: даже если anti-bot тормозит fallback, + # watchdog видит живую задачу и не убивает её как stalled. + self._report_progress( + "browser_fallback_progress", + fallback_processed=processed_local, + fallback_total=len(url_slice), vehicle_index=global_idx, vehicle_total=total, ) + + # Если anti-bot уже активен (много fallback URL), снижаем burst-нагрузку. + if len(url_slice) >= 20: + time.sleep(random.uniform(0.2, 0.7)) + if page is None: page = self._get_page() if self.settings.block_resources: @@ -1507,7 +1592,7 @@ class IAAIScraper: page.goto( url, wait_until="commit", - timeout=self.settings.fallback_navigation_timeout_ms, + timeout=max(3_000, int(self.settings.fallback_navigation_timeout_ms)), ) except Exception as exc: if self._is_protection_or_network_error(exc): @@ -1523,12 +1608,6 @@ class IAAIScraper: continue try: - self._emit_progress( - "browser_fallback_parse_start", - vehicle_url=url, - vehicle_index=global_idx, - vehicle_total=total, - ) js_data: dict = {} try: js_data = page.evaluate(self._JS_EXTRACT) or {} @@ -1586,15 +1665,6 @@ class IAAIScraper: record = CarRecord.model_validate(db_record.model_dump(mode="json")) local_records.append(record) - self._emit_progress( - "browser_fallback_parse_done", - vehicle_url=url, - vehicle_index=global_idx, - vehicle_total=total, - brand=record.brand, - model=record.model, - year=record.year, - ) logger.debug( "[%d/%d] Parsed %s %s %s (fallback)", global_idx, total, record.brand, record.model, record.year or "?", @@ -1604,13 +1674,6 @@ class IAAIScraper: local_protection += 1 local_failed += 1 local_failures.append({"vehicle_url": url, "error": str(exc)}) - self._emit_progress( - "browser_fallback_parse_failed", - vehicle_url=url, - vehicle_index=global_idx, - vehicle_total=total, - error=str(exc), - ) logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) finally: if page is not None: @@ -1625,17 +1688,30 @@ class IAAIScraper: "protection_events": local_protection, } - # Playwright sync API использует greenlets, привязанные к потоку. - # Запуск в ThreadPoolExecutor вызывает greenlet crash. - # Обрабатываем все слайсы последовательно в текущем потоке. - for s in slices: - if not s: - continue - res = _process_slice(s) + # Одна страница — в главном потоке. + if n_pages == 1: + res = _process_slice(slices[0] if slices else []) records.extend(res["records"]) failures.extend(res["failures"]) cars_failed += res["cars_failed"] protection_events += res["protection_events"] + else: + # Несколько страниц — параллельно, каждый поток со своей Page. + # Таймаут 5 минут на весь fallback — если страницы зависли, не блокируем навсегда. + _fallback_timeout = max(300, len(fallback_urls) * 30) + with ThreadPoolExecutor(max_workers=n_pages) as executor: + futures = [executor.submit(_process_slice, s) for s in slices if s] + for fut in as_completed(futures, timeout=_fallback_timeout): + try: + res = fut.result(timeout=60) + except FuturesTimeoutError: + logger.error("Browser fallback thread timed out") + cars_failed += 1 + continue + records.extend(res["records"]) + failures.extend(res["failures"]) + cars_failed += res["cars_failed"] + protection_events += res["protection_events"] return { "records": records, @@ -1650,6 +1726,10 @@ class IAAIScraper: lane: str = "iaai_cars", parallel_tabs: int | None = None, ) -> dict: + # Runtime config может меняться на лету (добавили/убрали бренды, диапазоны и т.п.). + # Подхватываем обновление перед каждым batch, чтобы фильтрация применялась сразу. + self._reload_runtime_config() + trace_id = self._new_trace_id("sync-batch") started_at = time.perf_counter() num_workers = parallel_tabs or self.settings.parallel_tabs @@ -1657,6 +1737,7 @@ class IAAIScraper: cars_upserted = 0 cars_failed = 0 + cars_filtered = 0 images_upserted = 0 records: list[CarRecord] = [] failures: list[dict[str, str]] = [] @@ -1666,15 +1747,16 @@ class IAAIScraper: total = len(vehicle_urls) logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) - self._emit_progress( - "sync_batch_start", - batch_size=total, - parallel_workers=num_workers, - first_vehicle_url=vehicle_urls[0] if vehicle_urls else None, - last_vehicle_url=vehicle_urls[-1] if vehicle_urls else None, + self._report_progress( + "sync_batch_started", + batch_total=total, + batch_workers=num_workers, + http_ok=0, + http_fallback=0, + batch_records_ready=0, + batch_failures=0, ) - self._ensure_http_fast_path_session() cookie_header = self._cookie_header_for_context() user_agent = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " @@ -1683,9 +1765,14 @@ class IAAIScraper: # ── Фаза 1: HTTP fast-path ── fallback_urls: list[tuple[str, int]] = [] + processed_indices: set[int] = set() + fallback_indices: set[int] = set() def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: idx, url = idx_url + # Небольшой анти-бёрст джиттер: разносим старт запросов в пуле, + # чтобы N воркеров не били в одну TLS-/PX-волну. + time.sleep(random.uniform(0.0, 0.15)) last_exc: Exception | None = None for attempt in range(1 + self._HTTP_MAX_RETRIES): try: @@ -1701,61 +1788,125 @@ class IAAIScraper: return idx, last_exc # type: ignore[return-value] indexed_urls = list(enumerate(vehicle_urls)) + # Таймаут на весь HTTP-пул: (connect 5 + read 20) × retries × URLs / workers + запас. + # Если пул зависнет дольше — отпускаем зависшие потоки и уходим в fallback. + _per_url_budget = (5 + 20) * (1 + self._HTTP_MAX_RETRIES) + 5 + _http_phase_timeout = max(120, _per_url_budget * max(1, total // max(1, num_workers))) + http_processed = 0 with ThreadPoolExecutor(max_workers=min(num_workers, total)) as executor: - for idx, result in executor.map(_fetch_one_with_retry, indexed_urls): - if isinstance(result, Exception): - http_fallbacks += 1 - logger.debug( - "[%d/%d] HTTP fast-path failed for %s: %s", - idx + 1, total, vehicle_urls[idx], result, - ) - fallback_urls.append((vehicle_urls[idx], idx + 1)) - else: - records.append(result) - http_successes += 1 - logger.debug( - "[%d/%d] Parsed %s %s %s (raw HTTP)", - idx + 1, total, result.brand, result.model, result.year or "?", + try: + for idx, result in executor.map( + _fetch_one_with_retry, indexed_urls, timeout=_http_phase_timeout, + ): + processed_indices.add(idx) + http_processed += 1 + if isinstance(result, Exception): + http_fallbacks += 1 + logger.debug( + "[%d/%d] HTTP fast-path failed for %s: %s", + idx + 1, total, vehicle_urls[idx], result, + ) + fallback_urls.append((vehicle_urls[idx], idx + 1)) + fallback_indices.add(idx) + else: + records.append(result) + http_successes += 1 + logger.debug( + "[%d/%d] Parsed %s %s %s (raw HTTP)", + idx + 1, total, result.brand, result.model, result.year or "?", + ) + self._report_progress( + "sync_batch_http_progress", + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), ) + except FuturesTimeoutError: + logger.error( + "HTTP phase timed out after %ds; %d/%d processed, rest go to fallback", + _http_phase_timeout, http_successes + http_fallbacks, total, + ) + # Все ещё не обработанные URL → в fallback + for i, url in enumerate(vehicle_urls): + if i in processed_indices or i in fallback_indices: + continue + if (url, i + 1) not in fallback_urls: + fallback_urls.append((url, i + 1)) + fallback_indices.add(i) + http_fallbacks += 1 logger.info( "HTTP phase done: %d OK, %d fallback (%.1fs)", http_successes, http_fallbacks, time.perf_counter() - started_at, ) - self._emit_progress( + self._report_progress( "sync_batch_http_done", - batch_size=total, - http_successes=http_successes, - http_fallbacks=http_fallbacks, + batch_total=total, + http_processed=http_processed, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), ) # ── Фаза 2: браузерный fallback ── if fallback_urls: - fallback_pages = max( - 1, - min( - self._FALLBACK_PARALLEL_PAGES, - num_workers, - len(fallback_urls), - ), - ) logger.info( - "Fallback browser mode: %d/%d URLs, %d page(s)", + "Fallback browser mode: %d/%d URLs, single page", len(fallback_urls), total, - fallback_pages, ) - self._emit_progress( - "sync_batch_fallback_start", - fallback_count=len(fallback_urls), - batch_size=total, - first_fallback_url=fallback_urls[0][0] if fallback_urls else None, - fallback_pages=fallback_pages, + self._report_progress( + "sync_batch_fallback_started", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), ) - fb_results = self._browser_fallback_parallel(fallback_urls, total, fallback_pages) + fb_results = self._browser_fallback_parallel(fallback_urls, total, 1) records.extend(fb_results["records"]) cars_failed += fb_results["cars_failed"] protection_events += fb_results["protection_events"] failures.extend(fb_results["failures"]) + self._report_progress( + "sync_batch_fallback_done", + batch_total=total, + http_ok=http_successes, + http_fallback=http_fallbacks, + fallback_total=len(fallback_urls), + batch_records_ready=len(records), + batch_failures=cars_failed + len(failures), + protection_events=protection_events, + ) + + # ── Фаза 2.5: пост-фильтр по runtime_config ── + filters_cfg = self.runtime_config.filters + if records and not filters_cfg.is_empty(): + pre_filter_count = len(records) + records = [ + rec for rec in records + if filters_cfg.matches({ + "brand": rec.brand, + "model": rec.model, + "year": rec.year, + "body_type": rec.body_type, + "color": rec.color, + "drive": rec.drive, + "gearbox": rec.gearbox, + "price": rec.price, + "mileage": rec.mileage, + }) + ] + cars_filtered = pre_filter_count - len(records) + if cars_filtered: + logger.info( + "Runtime filter: %d/%d records filtered out before DB upsert", + cars_filtered, pre_filter_count, + ) # ── Фаза 3: запись в БД ── if records: @@ -1771,29 +1922,63 @@ class IAAIScraper: records = unique_records try: + self._report_progress( + "sync_batch_db_upsert_started", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) + _db_start = time.perf_counter() batch_result = self.persistence.upsert_cars_batch(records) + _db_elapsed = time.perf_counter() - _db_start cars_upserted = batch_result["inserted"] + batch_result["updated"] images_upserted = batch_result["images_upserted"] + if _db_elapsed > 10: + logger.warning("DB upsert slow: %.1fs for %d records", _db_elapsed, len(records)) + self._report_progress( + "sync_batch_db_upsert_done", + batch_total=total, + batch_records_ready=len(records), + cars_upserted=cars_upserted, + images_upserted=images_upserted, + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + ) except Exception as exc: logger.error("Batch upsert failed: %s", exc) cars_failed += len(records) + self._report_progress( + "sync_batch_db_upsert_failed", + batch_total=total, + batch_records_ready=len(records), + http_ok=http_successes, + http_fallback=http_fallbacks, + batch_failures=cars_failed + len(failures), + error=str(exc), + ) status = "success" if not failures else ("partial_success" if cars_upserted else "failed") - self._emit_progress( + self._report_progress( "sync_batch_done", - batch_size=total, + batch_total=total, cars_upserted=cars_upserted, cars_failed=cars_failed, + cars_filtered=cars_filtered, images_upserted=images_upserted, - http_successes=http_successes, - http_fallbacks=http_fallbacks, - status=status, + http_ok=http_successes, + http_fallback=http_fallbacks, + protection_events=protection_events, + batch_failures=len(failures), ) return { "trace_id": trace_id, "status": status, "cars_upserted": cars_upserted, "cars_failed": cars_failed, + "cars_filtered": cars_filtered, "images_upserted": images_upserted, "http_successes": http_successes, "http_fallbacks": http_fallbacks, @@ -1814,6 +1999,10 @@ class IAAIScraper: year_max: int | None = None, skip_mark_sold: bool = False, ): + # Подхватываем актуальный runtime_config на каждый запуск sync, + # чтобы добавленные/удалённые бренды/модели применялись без рестарта. + self._reload_runtime_config() + # runtime_config — дефолты; CLI/API аргументы приоритетнее. rc = self.runtime_config.sync if limit is None and rc.limit is not None: @@ -1830,6 +2019,7 @@ class IAAIScraper: cars_upserted = 0 cars_failed = 0 cars_filtered = 0 + protection_events = 0 images_upserted = 0 total = 0 skipped_existing = 0 @@ -1839,42 +2029,24 @@ class IAAIScraper: try: effective_only_new = self.settings.sync_only_new if only_new is None else only_new - if self._should_use_sitemap_discovery( + stream_result = self._sync_listing_streaming( make=make, model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + started_at=started_at, listing_url=listing_url, year_min=year_min, year_max=year_max, - limit=limit, - effective_only_new=effective_only_new, - ): - try: - stream_result = self._sync_listing_via_sitemap( - lane=lane, - limit=limit, - effective_only_new=effective_only_new, - ) - except SitemapDiscoveryError as exc: - logger.error("Sitemap-only full scan failed: %s", exc) - raise - else: - stream_result = self._sync_listing_streaming( - make=make, - model=model, - lane=lane, - limit=limit, - effective_only_new=effective_only_new, - started_at=started_at, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - ) + ) listing = stream_result["listing"] total = stream_result["total"] skipped_existing = stream_result["skipped_existing"] cars_upserted = stream_result["cars_upserted"] cars_failed = stream_result["cars_failed"] images_upserted = stream_result["images_upserted"] + protection_events = int(stream_result.get("protection_events", 0)) failures.extend(stream_result["failures"]) all_listing_origin_urls = stream_result["all_listing_origin_urls"] @@ -1886,10 +2058,9 @@ class IAAIScraper: or (limit is not None and limit > 0) or listing.get("early_stopped", False) or listing.get("truncated_by_time_budget", False) - or listing.get("pagination_interrupted", False) ) if skip_mark_sold: - logger.debug("Skipping mark_sold: caller requested (parallel segment mode)") + logger.debug("Skipping mark_sold: caller requested") elif all_listing_origin_urls and not is_partial_scan: try: sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) @@ -1921,19 +2092,31 @@ class IAAIScraper: "Sync run #%d finished: %d/%d upserted, %d failed, %d filtered, %d images", run_id, cars_upserted, total, cars_failed, cars_filtered, images_upserted, ) + discovered_total = max(0, int(total)) + fail_ratio = (cars_failed / discovered_total) if discovered_total > 0 else 0.0 + protection_ratio = (protection_events / discovered_total) if discovered_total > 0 else 0.0 + anti_bot_detected = discovered_total > 0 and ( + (protection_events >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) return { "trace_id": trace_id, "status": status, "run_id": run_id, # partial_success допустим — отдельные машины могли не спарситься, # это не повод повторять весь bootstrap. - "full_scan_completed": (not is_partial_scan) and status in ("success", "partial_success"), + "full_scan_completed": (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected, "only_new_effective": effective_only_new, "listing": listing, + "total_discovered": discovered_total, "cars_upserted": cars_upserted, "cars_failed": cars_failed, "cars_filtered": cars_filtered, "images_upserted": images_upserted, + "protection_events": protection_events, + "anti_bot_detected": anti_bot_detected, + "fail_ratio": round(fail_ratio, 4), + "protection_ratio": round(protection_ratio, 4), "skipped_existing": skipped_existing, "elapsed_seconds": round(time.perf_counter() - started_at, 3), "failures": failures, @@ -1969,6 +2152,11 @@ class IAAIScraper: all_failures: list[dict[str, str]] = [] segment_results: list[dict[str, Any]] = [] completed_all = True + skipped_segments = 0 + + # В segmented-режиме полный прогон должен проходить ВСЕ сегменты. + # Runtime-фильтры применяются позже (на уровне конкретных карточек), + # но не должны сужать сам обход сегментов. logger.warning( "Starting segmented sync: %d segments, resume from segment=%d", @@ -1980,6 +2168,11 @@ class IAAIScraper: seg_make = seg.get("make") seg_year_min = seg.get("year_min") seg_year_max = seg.get("year_max") + + # На длительном full-scan сегмент нельзя пропускать из-за runtime include.brands, + # иначе прогон становится частичным. + self._reload_runtime_config() + seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None seg_label = f"{seg_make or 'ALL'}" @@ -1991,7 +2184,29 @@ class IAAIScraper: seg_idx + 1, len(segments), seg_label, ) + outer_progress_callback = self._progress_callback + + def _segment_progress(stage: str, meta: dict[str, Any]) -> None: + if outer_progress_callback is None: + return + outer_progress_callback( + stage, + { + **meta, + "segment_index": seg_idx, + "segment_label": seg_label, + "segments_total": len(segments), + }, + ) + try: + self.set_progress_callback(_segment_progress) + self._report_progress( + "segment_started", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + ) result = self.sync_listing( make=None if seg_url else seg_make, model=None, @@ -2018,11 +2233,18 @@ class IAAIScraper: }) segment_done = bool(result.get("full_scan_completed", False)) + # Даже если сегмент завершился неполно (например, страница/пагинация сломалась), + # в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше. if not segment_done: - completed_all = False + skipped_segments += 1 + logger.warning( + "Segment %d/%d incomplete: %s — advancing checkpoint and continuing", + seg_idx + 1, len(segments), seg_label, + ) - # Сегмент полностью пройден — фиксируем чекпоинт, даже если часть машин failed. - if segment_done and progress_callback is not None: + # Сегмент обработан до конечного состояния — фиксируем чекпоинт, + # даже если он был пропущен/оборван с ошибками. + if progress_callback is not None: try: progress_callback(seg_idx) except Exception: @@ -2038,12 +2260,46 @@ class IAAIScraper: result.get("cars_failed", 0), result.get("listing", {}).get("vehicles_collected", 0), ) + self._report_progress( + "segment_done", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + segment_status=result.get("status"), + segment_full_scan_completed=segment_done, + ) except Exception as exc: logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc) all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)}) - completed_all = False + skipped_segments += 1 + segment_results.append({ + "segment": seg, + "segment_index": seg_idx, + "status": "skipped_error", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "vehicles_collected": 0, + }) + if progress_callback is not None: + try: + progress_callback(seg_idx) + except Exception: + logger.warning( + "Failed to persist skipped segment checkpoint for segment=%d", + seg_idx, exc_info=True, + ) + self._report_progress( + "segment_failed", + segment_index=seg_idx, + segment_label=seg_label, + segments_total=len(segments), + error=str(exc), + ) # Продолжаем оставшиеся сегменты — одна ошибка не должна убивать весь прогон continue + finally: + self.set_progress_callback(outer_progress_callback) elapsed = round(time.perf_counter() - started_at, 3) status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed" @@ -2062,6 +2318,7 @@ class IAAIScraper: "full_scan_completed": completed_all, "segments_total": len(segments), "segments_completed": len(segment_results), + "segments_skipped": skipped_segments, "cars_upserted": total_cars_upserted, "cars_failed": total_cars_failed, "images_upserted": total_images_upserted, diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py index b68143f..a1740f7 100644 --- a/iaai_scraper/storage/db.py +++ b/iaai_scraper/storage/db.py @@ -14,13 +14,6 @@ from .schemas import CarRecord logger = logging.getLogger("iaai_scraper.db") -def get_db_session() -> Session: - """Получить новую сессию базы данных.""" - settings = Settings() - persistence = PersistenceService(settings) - return persistence.session_factory() - - CAR_DB_FIELDS = { col.key for col in Car.__table__.columns if col.key not in ("id",) @@ -42,11 +35,16 @@ class PersistenceService: engine_kwargs["max_overflow"] = settings.database.max_overflow engine_kwargs["pool_pre_ping"] = True engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds + engine_kwargs["pool_timeout"] = 30 + # Таймауты запросов и блокировок. + engine_kwargs["connect_args"] = { + "options": "-c statement_timeout=120000 -c lock_timeout=30000" + } self.engine = create_engine(settings.database.url, **engine_kwargs) self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) def create_tables(self) -> None: - # В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции. + # Для SQLite можно create_all. is_sqlite = self.settings.database.url.startswith("sqlite") if not is_sqlite and not self.settings.database.auto_create_tables: return @@ -112,7 +110,7 @@ class PersistenceService: def get_existing_urls_and_ids( self, origin_urls: list[str], origin_ids: list[str], ) -> tuple[set[str], set[str]]: - # Загрузка существующих URL и origin_id. + # Загрузка URL и origin_id. if not origin_urls and not origin_ids: return set(), set() urls: set[str] = set() @@ -321,7 +319,7 @@ class PersistenceService: return {"inserted": inserted, "updated": updated, "images_upserted": images_total} def upsert_car(self, record: CarRecord): - # Вставка или обновление автомобиля по origin_id/origin_url. + # Вставка или обновление авто. payload = self._car_payload(record) images = [image.model_dump(mode="python") for image in record.images] with self.session_scope() as session: @@ -382,7 +380,7 @@ class PersistenceService: - Один DELETE по car_id IN (...) вместо удаления по одному. - Fallback на по-одному upsert если batch commit упал. """ - # ── Дедупликация записей внутри батча ── + # Дедупликация батча. seen_ids: dict[str, int] = {} unique_records: list[CarRecord] = [] for idx, r in enumerate(records): @@ -413,20 +411,20 @@ class PersistenceService: images_total = 0 with self.session_scope() as session: - # Получаем существующие записи chunked-запросами. + # Загружаем существующие записи. origin_ids = [r.origin_id for r in records if r.origin_id] origin_urls = [r.origin_url for r in records if r.origin_url] existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) - # Предзагружаем ВСЕ изображения для обновляемых машин одним запросом. + # Предзагружаем изображения. existing_car_ids = set() for record in records: car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) if car is not None: existing_car_ids.add(int(car.id)) - # Строим маппинг car_id → set(image_urls) для сравнения. + # Готовим map car_id -> image_urls. existing_images_map = self._load_existing_image_urls(session, existing_car_ids) new_cars: list[tuple[Car, list[dict]]] = [] @@ -448,7 +446,7 @@ class PersistenceService: car.last_seen_at = record.last_seen_at updated += 1 - # Проверяем, изменились ли изображения. + # Проверяем изменения картинок. new_image_urls = {img.get("fullres_image", "") for img in images} old_image_urls = existing_images_map.get(int(car.id), set()) if new_image_urls != old_image_urls: @@ -456,15 +454,15 @@ class PersistenceService: else: images_total += len(old_image_urls) - # Один flush для всех вставок. + # Один flush. session.flush() - # Добавляем изображения для новых автомобилей. + # Добавляем картинки новым авто. for car, images in new_cars: self._add_images(session, int(car.id), images) images_total += len(images) - # Массово обновляем изображения только для машин с изменёнными картинками. + # Обновляем только изменённые картинки. if update_cars_needing_images: update_ids = [int(car.id) for car, _ in update_cars_needing_images] for i in range(0, len(update_ids), _IN_CHUNK_SIZE): @@ -477,7 +475,7 @@ class PersistenceService: return {"inserted": inserted, "updated": updated, "images_upserted": images_total} def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]: - # Fallback на поштучный upsert. + # Запасной поштучный upsert. inserted = 0 updated = 0 images_total = 0 @@ -517,72 +515,107 @@ class PersistenceService: Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, что кардинально быстрее при больших объёмах (100K+ URLs). + Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент. """ if not active_origin_urls: return 0 + # Нормализация URL. + def _norm(url: str) -> str: + return url.replace("~", "-") + + normalized_urls = {_norm(u) for u in active_origin_urls} + is_postgres = "postgresql" in self.settings.database.url with self.session_scope() as session: - # Страховка от ложного mark_sold при битом/неполном full-scan: - # если текущий список активных URL аномально мал относительно уже активных машин в БД, - # ничего не помечаем проданным. - active_db_count = int(session.execute( - select(func.count()) - .select_from(Car) - .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like(f"{lane}:%")) - ).scalar_one() or 0) - - current_active_count = len(active_origin_urls) - if active_db_count >= 1000 and current_active_count < max(500, int(active_db_count * 0.25)): - logger.warning( - "Skipping mark_sold: suspiciously small active set (%d URLs vs %d active in DB)", - current_active_count, - active_db_count, - ) - return 0 - if is_postgres: - # Создаём временную таблицу с активными URL. - session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT PRIMARY KEY) ON COMMIT DROP")) + # Считаем кандидатов на sold. + total_active = session.execute( + text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'") + ).scalar() or 0 + + if total_active == 0: + return 0 + + # Временная таблица URL. + session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP")) session.execute(text("TRUNCATE _active_urls")) - # Вставляем активные URL чанками через executemany. - url_list = list(active_origin_urls) + # Вставляем URL чанками. + url_list = list(normalized_urls) for i in range(0, len(url_list), _IN_CHUNK_SIZE): chunk = url_list[i:i + _IN_CHUNK_SIZE] - session.execute( - text("INSERT INTO _active_urls (url) VALUES (:url) ON CONFLICT DO NOTHING"), - [{"url": url} for url in chunk], - ) + values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) + params = {f"u{j}": url for j, url in enumerate(chunk)} + session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params) - # Массовая пометка проданных в PostgreSQL. + # Индекс для JOIN. + session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)")) + + # Считаем будущие sold. + would_mark = session.execute(text(""" + SELECT count(*) + FROM cars c + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url + WHERE a.url IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'iaai:%%' + """)).scalar() or 0 + + # Защита от аномалии. + if total_active > 100 and would_mark > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + would_mark, total_active, would_mark / total_active * 100, + ) + return 0 + + # Массовая пометка sold. result = session.execute(text(""" UPDATE cars SET is_sold = TRUE FROM ( SELECT c.id FROM cars c - LEFT JOIN _active_urls a ON c.origin_url = a.url + LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url WHERE a.url IS NULL AND c.is_sold = FALSE - AND c.origin_id LIKE :lane_prefix + AND c.origin_id LIKE 'iaai:%%' ) sub WHERE cars.id = sub.id - """), {"lane_prefix": f"{lane}:%"}) + """)) count = result.rowcount or 0 else: # Упрощённый путь для SQLite. stmt = ( update(Car) - .where(Car.origin_url.notin_(active_origin_urls)) .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like(f"{lane}:%")) + .where(Car.origin_id.like("iaai:%")) .values(is_sold=True) ) - result = session.execute(stmt) - count = result.rowcount or 0 + # Загружаем active URL. + all_active = session.execute( + select(Car.id, Car.origin_url).where( + Car.is_sold == False, Car.origin_id.like("iaai:%") # noqa: E712 + ) + ).all() + mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls] + + if not mark_ids: + return 0 + total_active = len(all_active) + if total_active > 100 and len(mark_ids) > total_active * 0.8: + logger.error( + "mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch", + len(mark_ids), total_active, len(mark_ids) / total_active * 100, + ) + return 0 + + for i in range(0, len(mark_ids), _IN_CHUNK_SIZE): + chunk = mark_ids[i:i + _IN_CHUNK_SIZE] + session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True)) + count = len(mark_ids) if count: logger.info("Marked %d cars as sold by URL (no longer in listing)", count) @@ -600,40 +633,43 @@ class PersistenceService: return {str(row[0]) for row in result if row and row[0]} def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]: - """Возвращает все активные (не sold) origin_url для указанного lane/prefix.""" + """Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса.""" with self.session_scope() as session: result = session.execute( - select(Car.origin_url) - .where(Car.origin_id.like(f"{prefix}%")) - .where(Car.is_sold == False) # noqa: E712 - .execution_options(yield_per=10000) + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).execution_options(yield_per=10000) ) return {str(row[0]) for row in result if row and row[0]} + def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int: + """Возвращает количество активных (не проданных) авто для заданного lane-префикса.""" + from sqlalchemy import func as sa_func + with self.session_scope() as session: + result = session.execute( + select(sa_func.count()).select_from(Car).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ) + ) + return int(result.scalar() or 0) + def get_active_origin_urls_batch_for_refresh( self, - *, prefix: str = "iaai:", offset: int = 0, - limit: int = 3000, + limit: int = 500, ) -> list[str]: - """Возвращает батч активных origin_url для циклического hourly refresh.""" - with self.session_scope() as session: - rows = session.execute( - select(Car.origin_url) - .where(Car.origin_id.like(f"{prefix}%")) - .where(Car.is_sold == False) # noqa: E712 - .order_by(Car.last_seen_at.asc(), Car.id.asc()) - .offset(max(0, int(offset))) - .limit(max(1, int(limit))) - ).all() - return [str(row[0]) for row in rows if row and row[0]] + """Возвращает батч origin_url активных авто для rolling refresh. - def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int: + Сортировка по last_seen_at ASC — давно не обновлённые идут первыми. + """ with self.session_scope() as session: - return int(session.execute( - select(func.count()) - .select_from(Car) - .where(Car.origin_id.like(f"{prefix}%")) - .where(Car.is_sold == False) # noqa: E712 - ).scalar_one() or 0) \ No newline at end of file + result = session.execute( + select(Car.origin_url).where( + Car.origin_id.like(f"{prefix}%"), + Car.is_sold == False, # noqa: E712 + ).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit) + ) + return [str(row[0]) for row in result if row and row[0]] \ No newline at end of file diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py index b9168d4..29d5566 100644 --- a/iaai_scraper/storage/models.py +++ b/iaai_scraper/storage/models.py @@ -80,65 +80,3 @@ class SyncRun(Base): cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) - - -class VehicleCandidate(Base): - __tablename__ = "vehicle_candidates" - __table_args__ = ( - Index("ix_vehicle_candidates_url", "url"), - Index("ix_vehicle_candidates_status_discovered", "status", "discovered_at"), - ) - id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) - url: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) - discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) - status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True) # pending, processing, processed, failed - priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - raw_snapshots: Mapped[list["VehicleRawSnapshot"]] = relationship("VehicleRawSnapshot", back_populates="candidate", cascade="all, delete-orphan") - - -class VehicleRawSnapshot(Base): - __tablename__ = "vehicle_raw_snapshots" - __table_args__ = ( - Index("ix_vehicle_raw_snapshots_candidate_id", "candidate_id"), - Index("ix_vehicle_raw_snapshots_captured_at", "captured_at"), - ) - id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) - candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True) - candidate: Mapped[VehicleCandidate] = relationship("VehicleCandidate", back_populates="raw_snapshots") - captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) - method: Mapped[str] = mapped_column(String(20), nullable=False) # http, browser - success: Mapped[bool] = mapped_column(Boolean, nullable=False) - raw_data: Mapped[str | None] = mapped_column(Text, nullable=True) # HTML or JSON content - error_message: Mapped[str | None] = mapped_column(Text, nullable=True) - parse_results: Mapped[list["VehicleParseResult"]] = relationship("VehicleParseResult", back_populates="snapshot", cascade="all, delete-orphan") - - -class VehicleParseResult(Base): - __tablename__ = "vehicle_parse_results" - __table_args__ = ( - Index("ix_vehicle_parse_results_snapshot_id", "snapshot_id"), - Index("ix_vehicle_parse_results_parsed_at", "parsed_at"), - ) - id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) - snapshot_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True) - snapshot: Mapped[VehicleRawSnapshot] = relationship("VehicleRawSnapshot", back_populates="parse_results") - parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) - success: Mapped[bool] = mapped_column(Boolean, nullable=False) - parsed_data: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON with parsed vehicle data - error_message: Mapped[str | None] = mapped_column(Text, nullable=True) - - -class VehicleRetryQueue(Base): - __tablename__ = "vehicle_retry_queue" - __table_args__ = ( - Index("ix_vehicle_retry_queue_candidate_id", "candidate_id"), - Index("ix_vehicle_retry_queue_retry_at", "retry_at"), - ) - id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) - candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True) - reason: Mapped[str] = mapped_column(String(50), nullable=False) # parse_failed, capture_failed, etc. - retry_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3) diff --git a/iaai_scraper/worker/celery_app.py b/iaai_scraper/worker/celery_app.py index 7ccabb1..66363f2 100644 --- a/iaai_scraper/worker/celery_app.py +++ b/iaai_scraper/worker/celery_app.py @@ -68,7 +68,7 @@ celery_app.conf.update( task_track_started=True, worker_concurrency=settings.celery.worker_concurrency, worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child, - worker_pool="prefork", + worker_pool="solo", worker_prefetch_multiplier=1, broker_connection_retry_on_startup=True, broker_transport_options={ @@ -82,8 +82,11 @@ celery_app.conf.update( "task": "iaai_scraper.worker.tasks.sync_listing_task", "schedule": settings.celery.beat_sync_interval_minutes * 60.0, "args": (), - "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False}, - "options": {"queue": "scraping"}, + "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True}, + "options": { + "queue": "scraping", + "expires": settings.celery.beat_sync_interval_minutes * 60.0, + }, } }, task_routes={ @@ -96,8 +99,8 @@ celery_app.autodiscover_tasks(["iaai_scraper.worker"]) @worker_ready.connect def _on_worker_ready(**kwargs): - """Сразу при старте worker отправляем первую задачу sync_listing, - чтобы не ждать час до первого beat-цикла.""" + """При старте worker отправляем первый sync_listing, если очередь пуста.""" + redis_client = None try: redis_client = Redis.from_url( settings.redis.url, @@ -107,10 +110,34 @@ def _on_worker_ready(**kwargs): health_check_interval=settings.redis.health_check_interval_seconds, retry_on_timeout=True, ) + + for stale_key in ("iaai:locks:sync_listing",): + try: + ttl = redis_client.ttl(stale_key) + if ttl is not None and ttl != -2: + redis_client.delete(stale_key) + logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl) + except Exception: + logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True) + + try: + queue_len = int(redis_client.llen("scraping") or 0) + except Exception: + queue_len = 0 + if queue_len > 0: + logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len) + return + should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) except Exception: logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True) return + finally: + if redis_client is not None: + try: + redis_client.close() + except Exception: + pass if not should_dispatch: logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue") @@ -121,4 +148,5 @@ def _on_worker_ready(**kwargs): "iaai_scraper.worker.tasks.sync_listing_task", kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False}, queue="scraping", - ) \ No newline at end of file + expires=settings.celery.beat_sync_interval_minutes * 60.0, + ) diff --git a/iaai_scraper/worker/self_heal.py b/iaai_scraper/worker/self_heal.py new file mode 100644 index 0000000..d337032 --- /dev/null +++ b/iaai_scraper/worker/self_heal.py @@ -0,0 +1,201 @@ +import json +import logging +import os +import random +import signal +import time + +from redis import Redis + + +logger = logging.getLogger("iaai_scraper.worker.self_heal") + +GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts" +SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress" + + +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _env_int(name: str, default: int) -> int: + value = os.getenv(name) + if value is None: + return default + try: + return int(value.strip()) + except Exception: + return default + + +def _get_redis() -> Redis: + url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0") + return Redis.from_url( + url, + decode_responses=True, + socket_connect_timeout=5.0, + socket_timeout=10.0, + health_check_interval=30, + retry_on_timeout=True, + ) + + +def _safe_int(value: str | None, default: int = 0) -> int: + if value is None: + return default + try: + return int(str(value).strip()) + except Exception: + return default + + +def _read_last_progress_ts(redis_client: Redis) -> int | None: + raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY) + if raw: + ts = _safe_int(raw) + if ts > 0: + return ts + + # Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*. + # Это дороже, но выполняется только при отсутствии основного маркера. + max_ts = 0 + for key in redis_client.scan_iter(match="iaai:state:task_progress:*"): + try: + payload = redis_client.get(key) + if not payload: + continue + data = json.loads(payload) + ts = _safe_int(data.get("ts"), 0) + if ts > max_ts: + max_ts = ts + except Exception: + continue + return max_ts or None + + +def _kill_worker_process() -> None: + pid_file = "/tmp/celery-worker.pid" + pid: int | None = None + try: + with open(pid_file, "r", encoding="utf-8") as f: + pid = int(f.read().strip()) + except Exception: + pid = None + + if not pid: + logger.error("Self-heal: failed to read worker pid from %s", pid_file) + return + + logger.error("Self-heal: terminating stuck worker process pid=%s", pid) + try: + os.kill(pid, signal.SIGTERM) + except Exception: + logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid) + return + + time.sleep(20) + try: + # Если процесс ещё жив — принудительно убиваем. + os.kill(pid, 0) + logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid) + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + except Exception: + logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid) + + +def main() -> None: + if not _env_bool("IAAI_SELF_HEAL_ENABLED", True): + logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED") + return + + queue_name = os.getenv("IAAI_CELERY_QUEUE", "scraping") + check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30)) + stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720)) + startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300)) + restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300)) + + logger.warning( + "Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss", + queue_name, + check_interval, + stall_seconds, + startup_grace, + restart_cooldown, + ) + + started_at = time.time() + redis_client: Redis | None = None + + while True: + try: + if redis_client is None: + redis_client = _get_redis() + redis_client.ping() + + queue_len = _safe_int(redis_client.llen(queue_name), 0) + if queue_len <= 0: + time.sleep(check_interval) + continue + + last_progress_ts = _read_last_progress_ts(redis_client) + now_ts = int(time.time()) + age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts)) + + if age is None: + if now_ts - int(started_at) < startup_grace: + time.sleep(check_interval) + continue + logger.warning( + "Self-heal: queue=%d but no progress timestamp found after startup grace", + queue_len, + ) + age = stall_seconds + 1 + + if age <= stall_seconds: + time.sleep(check_interval) + continue + + # Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно. + acquired = bool( + redis_client.set( + SELF_HEAL_RESTART_LOCK_KEY, + str(now_ts), + nx=True, + ex=restart_cooldown, + ) + ) + if not acquired: + time.sleep(check_interval) + continue + + logger.error( + "Self-heal: detected global stall (queue=%d, progress_age=%ss > %ss). Restarting worker process...", + queue_len, + age, + stall_seconds, + ) + # Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров + # перезапуск был не строго одновременно. + time.sleep(random.uniform(0.3, 2.0)) + _kill_worker_process() + # После kill pid1 контейнер будет перезапущен Docker restart-policy. + # На случай неуспеха не молотим цикл. + time.sleep(check_interval) + + except Exception: + logger.exception("Self-heal watchdog iteration failed") + redis_client = None + time.sleep(check_interval) + + +if __name__ == "__main__": + logging.basicConfig( + level=os.getenv("IAAI_LOG_LEVEL", "INFO"), + format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + ) + main() diff --git a/iaai_scraper/worker/tasks.py b/iaai_scraper/worker/tasks.py index d67185e..2954ff5 100644 --- a/iaai_scraper/worker/tasks.py +++ b/iaai_scraper/worker/tasks.py @@ -1,4 +1,4 @@ -# Задачи Celery для синхронизации автомобилей и листинга IAAI. +# Задачи Celery. import json import logging @@ -7,6 +7,7 @@ import signal from threading import Event, Thread import time import uuid +from typing import Callable from billiard.exceptions import SoftTimeLimitExceeded from celery import shared_task @@ -19,22 +20,57 @@ from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitema logger = logging.getLogger("iaai_scraper.worker.tasks") +# Пауза между батчами. +_INTER_BATCH_DELAY = max(float(os.getenv("IAAI_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0) +# Порог ошибок в батче. +_FAIL_RATE_THRESHOLD = min(max(float(os.getenv("IAAI_FAIL_RATE_THRESHOLD", "0.9")), 0.0), 1.0) + SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing" SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done" SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint" SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60 +SYNC_LISTING_RESUME_TARGET_KEY = "iaai:state:sync_listing_resume_target_segment" +SYNC_LISTING_RESUME_TARGET_STREAK_KEY = "iaai:state:sync_listing_resume_target_streak" +SYNC_LISTING_RESUME_TARGET_TTL_SECONDS = 24 * 60 * 60 +SYNC_LISTING_RESUME_TARGET_STREAK_LIMIT = max( + 1, + int(os.getenv("SYNC_LISTING_RESUME_TARGET_STREAK_LIMIT", "2")), +) +SYNC_LISTING_REPEAT_CHECKPOINT_KEY = "iaai:state:sync_listing_repeat_checkpoint" +SYNC_LISTING_REPEAT_CHECKPOINT_STREAK_KEY = "iaai:state:sync_listing_repeat_checkpoint_streak" +SYNC_LISTING_REPEAT_CHECKPOINT_TTL_SECONDS = 24 * 60 * 60 +SYNC_LISTING_REPEAT_CHECKPOINT_STREAK_LIMIT = max( + 1, + int(os.getenv("SYNC_LISTING_REPEAT_CHECKPOINT_STREAK_LIMIT", "2")), +) SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak" SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3 SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60 +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "iaai:state:sync_listing_bootstrap_continuation_streak" +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT = max( + 1, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT", "6")), +) +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS = max( + 60, + int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS", str(6 * 60 * 60))), +) +HOURLY_FAILURE_STREAK_KEY = "iaai:state:hourly_failure_streak" +HOURLY_FAILURE_STREAK_LIMIT = 3 +HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # Сброс через 6 часов. SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending" SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task" SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}" SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress" SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total" +SYNC_SEGMENTS_CYCLE_KEY = "iaai:state:sync_segments_cycle_id" SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60 TASK_PROGRESS_KEY_FMT = "iaai:state:task_progress:{task_id}" +GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts" SITEMAP_HOURLY_LAST_COUNT_KEY = "iaai:state:sitemap_hourly_last_count" SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "iaai:state:sitemap_hourly_refresh_offset" +SYNC_LISTING_STALLED_SEGMENT_KEY = "iaai:state:sync_listing_stalled_segment" +SYNC_LISTING_STALLED_SEGMENT_TTL_SECONDS = 24 * 60 * 60 def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): @@ -60,10 +96,7 @@ def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): def _run_browser_job(func, *args, **kwargs): - # С pool=solo Celery worker работает в одном процессе/потоке. - # Playwright sync API использует greenlets, которые привязаны к потоку. - # Запуск в отдельном потоке вызывает greenlet.error: cannot switch to a different thread. - # Поэтому запускаем напрямую в текущем потоке. + # Запускаем в текущем потоке. return func(*args, **kwargs) @@ -71,8 +104,7 @@ def _sync_listing_lock_ttl_seconds() -> int: settings = Settings() soft = settings.celery.task_soft_time_limit hard = settings.celery.task_time_limit - # Используем clamped hard limit (soft + 120), а не сырой task_time_limit, - # чтобы lock не висел 11 дней при CELERY_TASK_TIME_LIMIT=999999. + # Ограничиваем hard limit. effective_hard = min(hard, soft + 120) if soft else hard return max(effective_hard + 120, 300) @@ -94,17 +126,25 @@ def _update_task_progress( **payload, ) -> None: try: + now_ts = int(time.time()) data = { "task_id": task_id, "stage": stage, - "ts": int(time.time()), + "ts": now_ts, **payload, } - redis_client.set( + ttl = max(60, int(ttl_seconds)) + pipe = redis_client.pipeline() + pipe.set( _task_progress_key(task_id), json.dumps(data, ensure_ascii=False), - ex=max(60, int(ttl_seconds)), + ex=ttl, ) + # Глобальный маркер активности для внешнего guard-процесса. + # Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера + # (когда PID жив, но прогресс по задачам не двигается). + pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60)) + pipe.execute() except Exception: logger.warning("Failed to update task progress for %s", task_id, exc_info=True) @@ -116,7 +156,7 @@ def _clear_task_progress(redis_client: Redis, task_id: str) -> None: logger.warning("Failed to clear task progress for %s", task_id, exc_info=True) -def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | None) -> dict[str, object]: +def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: del limit, only_new settings = Settings() persistence = _get_persistence() @@ -134,19 +174,33 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | cars_upserted = 0 cars_failed = 0 + protection_events = 0 images_upserted = 0 failures: list[dict[str, str]] = [] if new_urls: with IAAIScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) batch_size = settings.celery.batch_size for batch_start in range(0, len(new_urls), batch_size): batch_urls = new_urls[batch_start:batch_start + batch_size] batch_result = scraper.sync_batch(batch_urls, lane=lane) - cars_upserted += int(batch_result.get("cars_upserted", 0)) - cars_failed += int(batch_result.get("cars_failed", 0)) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection images_upserted += int(batch_result.get("images_upserted", 0)) failures.extend(batch_result.get("failures", [])) + # Fail-fast: если слишком много ошибок — IAAI блокирует, не тратим ресурсы. + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(new_urls): + time.sleep(_INTER_BATCH_DELAY) status = "success" if not failures else ("partial_success" if cars_upserted else "failed") return { @@ -163,11 +217,12 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | "discovered_urls": len(discovered_urls), "new_urls": len(new_urls), "sold_marked": sold_count, + "protection_events": protection_events, "transport": discovery_result.stats.transport, } -def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: bool | None) -> dict[str, object]: +def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: bool | None, progress_callback=None) -> dict[str, object]: del limit, only_new settings = Settings() persistence = _get_persistence() @@ -182,18 +237,31 @@ def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: cars_upserted = 0 cars_failed = 0 + protection_events = 0 images_upserted = 0 failures: list[dict[str, str]] = [] with IAAIScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) batch_size = settings.celery.batch_size for batch_start in range(0, len(discovered_urls), batch_size): batch_urls = discovered_urls[batch_start:batch_start + batch_size] batch_result = scraper.sync_batch(batch_urls, lane=lane) - cars_upserted += int(batch_result.get("cars_upserted", 0)) - cars_failed += int(batch_result.get("cars_failed", 0)) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection images_upserted += int(batch_result.get("images_upserted", 0)) failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + batch_size < len(discovered_urls): + time.sleep(_INTER_BATCH_DELAY) status = "success" if not failures else ("partial_success" if cars_upserted else "failed") return { @@ -210,6 +278,7 @@ def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: "discovered_urls": len(discovered_urls), "new_urls": None, "sold_marked": sold_count, + "protection_events": protection_events, "transport": discovery_result.stats.transport, } @@ -220,6 +289,7 @@ def _hourly_sitemap_rolling_refresh_sync( lane: str, limit: int | None, only_new: bool | None, + progress_callback=None, ) -> dict[str, object]: del limit, only_new settings = Settings() @@ -277,19 +347,32 @@ def _hourly_sitemap_rolling_refresh_sync( cars_upserted = 0 cars_failed = 0 + protection_events = 0 images_upserted = 0 failures: list[dict[str, str]] = [] if target_urls: with IAAIScraper(settings) as scraper: + if progress_callback: + scraper.set_progress_callback(progress_callback) worker_batch_size = settings.celery.batch_size for batch_start in range(0, len(target_urls), worker_batch_size): batch_urls = target_urls[batch_start:batch_start + worker_batch_size] batch_result = scraper.sync_batch(batch_urls, lane=lane) - cars_upserted += int(batch_result.get("cars_upserted", 0)) - cars_failed += int(batch_result.get("cars_failed", 0)) + batch_ok = int(batch_result.get("cars_upserted", 0)) + batch_fail = int(batch_result.get("cars_failed", 0)) + batch_protection = int(batch_result.get("protection_events", 0)) + cars_upserted += batch_ok + cars_failed += batch_fail + protection_events += batch_protection images_upserted += int(batch_result.get("images_upserted", 0)) failures.extend(batch_result.get("failures", [])) + total_in_batch = batch_ok + batch_fail + if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: + logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) + break + if batch_start + worker_batch_size < len(target_urls): + time.sleep(_INTER_BATCH_DELAY) status = "success" if not failures else ("partial_success" if cars_upserted else "failed") return { @@ -307,6 +390,7 @@ def _hourly_sitemap_rolling_refresh_sync( "new_urls": len(new_urls), "refresh_urls": len(refresh_urls), "sold_marked": sold_count, + "protection_events": protection_events, "transport": discovery_result.stats.transport, "refresh_offset": offset, "refresh_next_offset": next_offset, @@ -319,61 +403,159 @@ def _start_stall_watchdog( *, task_id: str, stall_timeout_seconds: int, + lock_key: str | None = None, + lock_owner: str | None = None, + on_stall: Callable[[dict[str, object]], None] | None = None, ) -> tuple[Event, Thread]: stop_event = Event() interval_seconds = max(5.0, min(30.0, stall_timeout_seconds / 3)) def _watchdog() -> None: key = _task_progress_key(task_id) + no_data_count = 0 + # Абсолютный дедлайн: если watchdog работает дольше 3× stall_timeout без прогресса — убиваем. + watchdog_born = time.monotonic() + absolute_deadline = stall_timeout_seconds * 3 while not stop_event.wait(interval_seconds): try: raw = redis_client.get(key) if not raw: - continue - data = json.loads(raw) - last_ts = int(data.get("ts") or 0) - if not last_ts: - continue - age = int(time.time()) - last_ts - if age < stall_timeout_seconds: - continue - logger.error( - "Task %s stalled for %ss at stage=%s payload=%s; killing worker process for redelivery", - task_id, - age, - data.get("stage"), - data, - ) + no_data_count += 1 + elapsed_since_born = time.monotonic() - watchdog_born + if no_data_count % 5 == 0: + logger.warning( + "Stall watchdog: no progress data for task %s after %d checks (%.0fs)", + task_id, no_data_count, elapsed_since_born, + ) + # Если прогресс-данных нет дольше stall_timeout — считаем задачу мёртвой. + if elapsed_since_born > stall_timeout_seconds: + logger.error( + "Task %s has no progress data for %.0fs (> %ds); treating as stalled", + task_id, elapsed_since_born, stall_timeout_seconds, + ) + else: + continue + else: + no_data_count = 0 + data = json.loads(raw) + last_ts = int(data.get("ts") or 0) + if not last_ts: + continue + age = int(time.time()) - last_ts + if age < stall_timeout_seconds: + watchdog_born = time.monotonic() # reset absolute deadline on real progress + continue + logger.error( + "Task %s stalled for %ss at stage=%s payload=%s; cleaning up and restarting", + task_id, + age, + data.get("stage"), + data, + ) + if on_stall is not None: + try: + on_stall(data) + except Exception: + logger.warning("Stall watchdog hook failed for task %s", task_id, exc_info=True) except Exception: logger.warning("Failed to inspect task progress for stall watchdog", exc_info=True) - continue - os.kill(os.getpid(), signal.SIGKILL) + # Если Redis тоже не отвечает дольше дедлайна — убиваем. + if time.monotonic() - watchdog_born > absolute_deadline: + logger.error("Stall watchdog: Redis unreachable for %.0fs; forcing kill", time.monotonic() - watchdog_born) + else: + continue + + # ── Pre-SIGTERM cleanup: release lock so next task can run ── + if lock_key and lock_owner: + try: + _release_lock_if_owner(redis_client, lock_key, lock_owner) + logger.info("Stall watchdog: released lock %s before SIGTERM", lock_key) + except Exception: + # Force-delete if owner check fails (process is dying anyway) + try: + redis_client.delete(lock_key) + logger.info("Stall watchdog: force-deleted lock %s", lock_key) + except Exception: + logger.warning("Stall watchdog: failed to release lock %s", lock_key, exc_info=True) + + # ── Queue a followup task so parsing resumes after restart ── + try: + followup_ttl = max(180, int(stall_timeout_seconds) + 300) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + from ..worker.celery_app import celery_app + celery_app.send_task( + SYNC_LISTING_TASK_NAME, + kwargs={}, + queue="scraping", + countdown=15, + expires=followup_ttl, + ) + logger.info("Stall watchdog: queued followup sync_listing_task after stall") + else: + logger.info("Stall watchdog: followup already pending, skip duplicate enqueue") + except Exception: + try: + _clear_followup_pending(redis_client) + except Exception: + pass + logger.warning("Stall watchdog: failed to queue followup task", exc_info=True) + + # SIGTERM даёт процессу время на cleanup (закрыть DB, browser). + # Celery перехватит SIGTERM и поднимет Terminated / warm shutdown. + try: + os.kill(os.getpid(), signal.SIGTERM) + except OSError: + pass + # Даём 30 секунд на graceful shutdown, потом SIGKILL как последний resort. + stop_event.wait(30) + if not stop_event.is_set(): + logger.error("Task %s did not stop after SIGTERM; forcing SIGKILL", task_id) + os.kill(os.getpid(), signal.SIGKILL) thread = Thread(target=_watchdog, name=f"task-stall-watchdog-{task_id[:8]}", daemon=True) thread.start() return stop_event, thread +_persistence_instance: PersistenceService | None = None + + def _get_persistence() -> PersistenceService: - settings = Settings() - persistence = PersistenceService(settings) + global _persistence_instance + if _persistence_instance is not None: + return _persistence_instance + + sett = Settings() + persistence = PersistenceService(sett) def _ping_db() -> None: with persistence.engine.connect() as conn: conn.exec_driver_sql("SELECT 1") _retry_with_backoff(_ping_db, attempts=5, base_delay_s=1.0) + _persistence_instance = persistence return persistence +_redis_instance: Redis | None = None + + def _get_redis() -> Redis: - settings = Settings() + global _redis_instance + if _redis_instance is not None: + try: + _redis_instance.ping() + return _redis_instance + except Exception: + _redis_instance = None + + sett = Settings() redis_client = Redis.from_url( - settings.redis.url, + sett.redis.url, decode_responses=True, - socket_connect_timeout=settings.redis.socket_connect_timeout_seconds, - socket_timeout=settings.redis.socket_timeout_seconds, - health_check_interval=settings.redis.health_check_interval_seconds, + socket_connect_timeout=sett.redis.socket_connect_timeout_seconds, + socket_timeout=sett.redis.socket_timeout_seconds, + health_check_interval=sett.redis.health_check_interval_seconds, retry_on_timeout=True, ) @@ -381,6 +563,7 @@ def _get_redis() -> Redis: redis_client.ping() _retry_with_backoff(_ping_redis, attempts=5, base_delay_s=1.0) + _redis_instance = redis_client return redis_client @@ -548,6 +731,158 @@ def _clear_sync_checkpoint(redis_client: Redis) -> None: logger.warning("Failed to clear sync listing checkpoint", exc_info=True) +def _save_stalled_segment(redis_client: Redis, segment_index: int, *, task_id: str, stage: str | None) -> None: + try: + payload = { + "segment_index": int(segment_index), + "task_id": task_id, + "stage": stage, + "ts": int(time.time()), + } + redis_client.set( + SYNC_LISTING_STALLED_SEGMENT_KEY, + json.dumps(payload, ensure_ascii=False), + ex=SYNC_LISTING_STALLED_SEGMENT_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to persist stalled segment info", exc_info=True) + + +def _infer_stalled_segment_index(redis_client: Redis, payload: dict[str, object]) -> int | None: + """Пытается определить индекс застрявшего сегмента из payload или checkpoint.""" + raw_segment = payload.get("segment_index") + if raw_segment is not None: + try: + return max(0, int(raw_segment)) + except Exception: + pass + + last_completed = _load_last_completed_segment(redis_client) + if last_completed is None: + return 0 + return max(0, int(last_completed) + 1) + + +def _load_stalled_segment(redis_client: Redis) -> int | None: + try: + raw = redis_client.get(SYNC_LISTING_STALLED_SEGMENT_KEY) + except Exception: + logger.warning("Failed to read stalled segment info", exc_info=True) + return None + if not raw: + return None + try: + data = json.loads(raw) + return int(data.get("segment_index")) + except Exception: + logger.warning("Invalid stalled segment payload %r; clearing", raw) + _clear_stalled_segment(redis_client) + return None + + +def _clear_stalled_segment(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_STALLED_SEGMENT_KEY) + except Exception: + logger.warning("Failed to clear stalled segment info", exc_info=True) + + +def _register_bootstrap_resume_target(redis_client: Redis, segment_index: int) -> tuple[int, bool]: + """Возвращает (streak, should_skip_segment) для защиты от вечного resume на одном сегменте.""" + ttl = max(60, int(SYNC_LISTING_RESUME_TARGET_TTL_SECONDS)) + try: + raw_target = redis_client.get(SYNC_LISTING_RESUME_TARGET_KEY) + raw_streak = redis_client.get(SYNC_LISTING_RESUME_TARGET_STREAK_KEY) + prev_target = int(str(raw_target).strip()) if raw_target is not None else None + prev_streak = int(str(raw_streak).strip()) if raw_streak is not None else 0 + except Exception: + logger.warning("Failed to read bootstrap resume target guard", exc_info=True) + prev_target = None + prev_streak = 0 + + streak = (prev_streak + 1) if prev_target == int(segment_index) else 1 + + try: + pipe = redis_client.pipeline() + pipe.set(SYNC_LISTING_RESUME_TARGET_KEY, str(int(segment_index)), ex=ttl) + pipe.set(SYNC_LISTING_RESUME_TARGET_STREAK_KEY, str(int(streak)), ex=ttl) + pipe.execute() + except Exception: + logger.warning("Failed to persist bootstrap resume target guard", exc_info=True) + + # Если bootstrap второй раз подряд возвращается в тот же сегмент, + # считаем его застрявшим и отпускаем checkpoint дальше. + should_skip_segment = streak >= SYNC_LISTING_RESUME_TARGET_STREAK_LIMIT + if should_skip_segment: + logger.error( + "Bootstrap resume loop guard OPEN: segment=%d streak=%d limit=%d", + segment_index, + streak, + SYNC_LISTING_RESUME_TARGET_STREAK_LIMIT, + ) + elif streak > 1: + logger.warning( + "Bootstrap resume repeated for segment=%d (%d/%d)", + segment_index, + streak, + SYNC_LISTING_RESUME_TARGET_STREAK_LIMIT, + ) + return streak, should_skip_segment + + +def _register_repeated_bootstrap_checkpoint(redis_client: Redis, checkpoint_segment: int) -> tuple[int, bool]: + """Возвращает (streak, should_skip_next_segment), если bootstrap снова стартует с тем же checkpoint.""" + ttl = max(60, int(SYNC_LISTING_REPEAT_CHECKPOINT_TTL_SECONDS)) + try: + raw_checkpoint = redis_client.get(SYNC_LISTING_REPEAT_CHECKPOINT_KEY) + raw_streak = redis_client.get(SYNC_LISTING_REPEAT_CHECKPOINT_STREAK_KEY) + prev_checkpoint = int(str(raw_checkpoint).strip()) if raw_checkpoint is not None else None + prev_streak = int(str(raw_streak).strip()) if raw_streak is not None else 0 + except Exception: + logger.warning("Failed to read repeated bootstrap checkpoint guard", exc_info=True) + prev_checkpoint = None + prev_streak = 0 + + streak = (prev_streak + 1) if prev_checkpoint == int(checkpoint_segment) else 1 + + try: + pipe = redis_client.pipeline() + pipe.set(SYNC_LISTING_REPEAT_CHECKPOINT_KEY, str(int(checkpoint_segment)), ex=ttl) + pipe.set(SYNC_LISTING_REPEAT_CHECKPOINT_STREAK_KEY, str(int(streak)), ex=ttl) + pipe.execute() + except Exception: + logger.warning("Failed to persist repeated bootstrap checkpoint guard", exc_info=True) + + should_skip_next_segment = streak >= SYNC_LISTING_REPEAT_CHECKPOINT_STREAK_LIMIT + if should_skip_next_segment: + logger.error( + "Bootstrap checkpoint repeat guard OPEN: checkpoint=%d streak=%d limit=%d", + checkpoint_segment, + streak, + SYNC_LISTING_REPEAT_CHECKPOINT_STREAK_LIMIT, + ) + elif streak > 1: + logger.warning( + "Bootstrap repeated with same checkpoint=%d (%d/%d)", + checkpoint_segment, + streak, + SYNC_LISTING_REPEAT_CHECKPOINT_STREAK_LIMIT, + ) + return streak, should_skip_next_segment + + +def _clear_bootstrap_resume_target(redis_client: Redis) -> None: + try: + redis_client.delete( + SYNC_LISTING_RESUME_TARGET_KEY, + SYNC_LISTING_RESUME_TARGET_STREAK_KEY, + SYNC_LISTING_REPEAT_CHECKPOINT_KEY, + SYNC_LISTING_REPEAT_CHECKPOINT_STREAK_KEY, + ) + except Exception: + logger.warning("Failed to clear bootstrap resume target guard", exc_info=True) + + def _try_set_followup_pending(redis_client: Redis, *, ttl_seconds: int) -> bool: try: return bool(redis_client.set(SYNC_LISTING_FOLLOWUP_PENDING_KEY, "1", nx=True, ex=max(60, int(ttl_seconds)))) @@ -602,6 +937,70 @@ def _clear_bootstrap_failure_streak(redis_client: Redis) -> None: logger.warning("Failed to clear bootstrap failure streak", exc_info=True) +def _bump_bootstrap_continuation_streak(redis_client: Redis) -> tuple[int, bool]: + """Счётчик подряд идущих bootstrap-followup запусков. + + Возвращает (streak, should_continue). Если should_continue=False, + немедленные continuation блокируются до следующего beat-цикла. + """ + try: + streak = int(redis_client.incr(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY)) + redis_client.expire( + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to update bootstrap continuation streak", exc_info=True) + return 0, True + + should_continue = streak <= SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT + if not should_continue: + logger.error( + "Bootstrap continuation breaker OPEN: streak=%d limit=%d", + streak, + SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT, + ) + return streak, should_continue + + +def _clear_bootstrap_continuation_streak(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY) + except Exception: + logger.warning("Failed to clear bootstrap continuation streak", exc_info=True) + + +def _bump_hourly_failure_streak(redis_client: Redis) -> int: + """Инкрементирует счётчик ошибок hourly. Возвращает новое значение.""" + try: + streak = int(redis_client.incr(HOURLY_FAILURE_STREAK_KEY)) + redis_client.expire(HOURLY_FAILURE_STREAK_KEY, HOURLY_FAILURE_STREAK_TTL_SECONDS) + return streak + except Exception: + logger.warning("Failed to update hourly failure streak", exc_info=True) + return 0 + + +def _check_hourly_circuit_breaker(redis_client: Redis) -> tuple[int, bool]: + """Проверяет открыт ли circuit breaker. Возвращает (streak, is_open).""" + try: + raw = redis_client.get(HOURLY_FAILURE_STREAK_KEY) + streak = int(raw) if raw else 0 + except Exception: + return 0, False + is_open = streak >= HOURLY_FAILURE_STREAK_LIMIT + if is_open: + logger.error("Hourly circuit breaker OPEN (%d/%d failures) — skipping", streak, HOURLY_FAILURE_STREAK_LIMIT) + return streak, is_open + + +def _clear_hourly_failure_streak(redis_client: Redis) -> None: + try: + redis_client.delete(HOURLY_FAILURE_STREAK_KEY) + except Exception: + pass + + def _start_lock_heartbeat( redis_client: Redis, key: str, @@ -612,11 +1011,19 @@ def _start_lock_heartbeat( interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) def _heartbeat() -> None: + consecutive_failures = 0 while not stop_event.wait(interval_seconds): refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) if refreshed is False: logger.warning("Lost sync_listing lock ownership for %s", owner_token) return + if refreshed is None: + consecutive_failures += 1 + if consecutive_failures >= 5: + logger.error("Lock heartbeat failed %d times in a row for %s; giving up", consecutive_failures, owner_token) + return + else: + consecutive_failures = 0 thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True) thread.start() @@ -633,17 +1040,59 @@ def _reset_segments_progress(redis_client: Redis, total: int) -> None: logger.warning("Failed to reset segments progress", exc_info=True) +def _clear_segments_progress_state(redis_client: Redis) -> None: + try: + redis_client.delete( + SYNC_SEGMENTS_PROGRESS_KEY, + SYNC_SEGMENTS_TOTAL_KEY, + SYNC_SEGMENTS_CYCLE_KEY, + ) + except Exception: + logger.warning("Failed to clear segments progress state", exc_info=True) + + +def _ensure_segments_cycle(redis_client: Redis, total: int) -> tuple[str, bool]: + """Возвращает (cycle_id, created_new_cycle).""" + ttl = max(60, int(SYNC_SEGMENTS_PROGRESS_TTL_SECONDS)) + try: + raw_cycle = redis_client.get(SYNC_SEGMENTS_CYCLE_KEY) + cycle_id = str(raw_cycle).strip() if raw_cycle else "" + except Exception: + logger.warning("Failed to read segments cycle id", exc_info=True) + cycle_id = "" + + if cycle_id: + try: + pipe = redis_client.pipeline() + pipe.expire(SYNC_SEGMENTS_CYCLE_KEY, ttl) + pipe.expire(SYNC_SEGMENTS_PROGRESS_KEY, ttl) + pipe.set(SYNC_SEGMENTS_TOTAL_KEY, str(int(total)), ex=ttl) + pipe.execute() + except Exception: + logger.warning("Failed to refresh segments cycle ttl", exc_info=True) + return cycle_id, False + + cycle_id = uuid.uuid4().hex + try: + _reset_segments_progress(redis_client, total) + redis_client.set(SYNC_SEGMENTS_CYCLE_KEY, cycle_id, ex=ttl) + except Exception: + logger.warning("Failed to initialize segments cycle id", exc_info=True) + return cycle_id, True + + def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[int, int]: """Помечает сегмент завершённым. Возвращает (completed_count, total).""" try: pipe = redis_client.pipeline() pipe.sadd(SYNC_SEGMENTS_PROGRESS_KEY, str(int(segment_index))) pipe.expire(SYNC_SEGMENTS_PROGRESS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.expire(SYNC_SEGMENTS_CYCLE_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) pipe.scard(SYNC_SEGMENTS_PROGRESS_KEY) pipe.get(SYNC_SEGMENTS_TOTAL_KEY) results = pipe.execute() - completed = int(results[2] or 0) - total = int(results[3] or 0) if results[3] else 0 + completed = int(results[3] or 0) + total = int(results[4] or 0) if results[4] else 0 return completed, total except Exception: logger.warning("Failed to mark segment %d completed", segment_index, exc_info=True) @@ -708,6 +1157,8 @@ def sync_segment_task( redis_client, task_id=task_id, stall_timeout_seconds=stall_timeout, + lock_key=seg_lock_key, + lock_owner=owner_token, ) try: @@ -878,6 +1329,8 @@ def sync_listing_task( lock_ttl = _sync_listing_lock_ttl_seconds() heartbeat_stop: Event | None = None heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None force_bootstrap_full_scan = False def _enqueue_bootstrap_followup( @@ -900,7 +1353,28 @@ def sync_listing_task( return else: _clear_bootstrap_failure_streak(redis_client) + + continuation_streak, should_continue = _bump_bootstrap_continuation_streak(redis_client) + if not should_continue: + _clear_followup_pending(redis_client) + # Переключаемся на часовой beat-режим и останавливаем + # немедленные bootstrap continuation, чтобы не зациклиться. + if not always_full_scan: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + logger.error( + "Bootstrap continuation stopped after %d immediate runs; switching to hourly schedule", + continuation_streak, + ) + else: + logger.error( + "Bootstrap continuation stopped after %d immediate runs (always_full_scan=true)", + continuation_streak, + ) + return + try: + followup_expires = max(int(lock_ttl), int(delay_seconds) + 300) self.app.send_task( "iaai_scraper.worker.tasks.sync_listing_task", kwargs={ @@ -912,6 +1386,7 @@ def sync_listing_task( }, queue="scraping", countdown=max(0, int(delay_seconds)), + expires=followup_expires, ) logger.info( "Bootstrap follow-up sync queued in %ss (reason=%s)", @@ -939,19 +1414,69 @@ def sync_listing_task( try: settings = Settings() + + # Любой реально стартовавший sync_listing снимает pending-флаг followup, + # чтобы watchdog/continuation могли корректно планировать следующий run + # только при новой проблеме, а не копить дубликаты в очереди. + _clear_followup_pending(redis_client) + + # Start heartbeat + stall watchdog immediately after lock acquisition + # so ALL code paths (hourly, bootstrap, segmented) are protected. + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, + SYNC_LISTING_LOCK_KEY, + owner_token, + lock_ttl, + ) + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + lock_key=SYNC_LISTING_LOCK_KEY, + lock_owner=owner_token, + on_stall=( + lambda payload: ( + _save_stalled_segment( + redis_client, + stalled_segment_index, + task_id=task_id, + stage=str(payload.get("stage") or ""), + ) + if force_bootstrap_full_scan + and (stalled_segment_index := _infer_stalled_segment_index(redis_client, payload)) is not None + else None + ) + ), + ) + _update_task_progress( + redis_client, + task_id=task_id, + stage="sync_listing_started", + ttl_seconds=progress_ttl, + ) + full_scan_done_before_run = _is_full_scan_done(redis_client) - force_bootstrap_full_scan = not full_scan_done_before_run + always_full_scan = bool(settings.discovery.always_full_scan) + force_bootstrap_full_scan = always_full_scan or (not full_scan_done_before_run) effective_limit = None if force_bootstrap_full_scan else limit effective_only_new = False if force_bootstrap_full_scan else only_new hourly_mode = settings.discovery.hourly_mode.strip().lower() discovery_mode = settings.discovery.mode.strip().lower() + if always_full_scan: + # Для режима "полный прогон каждый запуск" приоритет — устойчивый resume, + # поэтому принудительно уходим в listing/segmented path вместо sitemap-mainline. + discovery_mode = "listing" prefer_sitemap_mainline = ( - make is None + not force_bootstrap_full_scan + and make is None and model is None and effective_limit is None and not effective_only_new + and not always_full_scan ) - use_hourly_sitemap_sync = full_scan_done_before_run and prefer_sitemap_mainline + use_hourly_sitemap_sync = (not always_full_scan) and full_scan_done_before_run and prefer_sitemap_mainline # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. # Используется только во время bootstrap для пропуска уже обработанных сегментов. @@ -959,14 +1484,20 @@ def sync_listing_task( last_completed_segment: int | None = None if force_bootstrap_full_scan: last_completed_segment = _load_last_completed_segment(redis_client) + stalled_segment = _load_stalled_segment(redis_client) else: # После завершения bootstrap чекпоинт не нужен никогда. _clear_sync_checkpoint(redis_client) + stalled_segment = None if force_bootstrap_full_scan: logger.info( "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run", ) + if always_full_scan: + logger.info( + "Always full scan mode enabled (IAAI_ALWAYS_FULL_SCAN=true): using listing resume path", + ) logger.info( "sync_listing options: only_new=%s, limit=%s", @@ -975,65 +1506,124 @@ def sync_listing_task( ) if use_hourly_sitemap_sync: - if hourly_mode == "diff": - logger.info("Hourly mode: running sitemap diff sync instead of full listing traversal") - result = _hourly_sitemap_diff_sync( - lane=lane, - limit=effective_limit, - only_new=effective_only_new, - ) - elif hourly_mode == "full_refresh": - logger.info("Hourly mode: running sitemap full refresh of all active vehicles") - result = _hourly_sitemap_full_refresh_sync( - lane=lane, - limit=effective_limit, - only_new=effective_only_new, - ) - else: - logger.info("Hourly mode: running sitemap rolling refresh of active vehicles") - result = _hourly_sitemap_rolling_refresh_sync( - redis_client=redis_client, - lane=lane, - limit=effective_limit, - only_new=effective_only_new, - ) - try: - redis_client.set(SITEMAP_HOURLY_LAST_COUNT_KEY, str(result.get("discovered_urls", 0))) - except Exception: - logger.warning("Failed to persist hourly sitemap count", exc_info=True) - - summary = { - "task_id": task_id, - "run_id": result.get("run_id"), - "status": result.get("status", "success"), - "cars_upserted": result.get("cars_upserted", 0), - "cars_failed": result.get("cars_failed", 0), - "images_upserted": result.get("images_upserted", 0), - "skipped_existing": result.get("skipped_existing", 0), - "elapsed_seconds": result.get("elapsed_seconds"), - "failures_count": len(result.get("failures") or []), - "hourly_mode": result.get("hourly_mode"), - "discovered_urls": result.get("discovered_urls", 0), - "new_urls": result.get("new_urls", 0), - "sold_marked": result.get("sold_marked", 0), - } - logger.info( - "sync_listing_task hourly diff completed: status=%s, new=%d, sold=%d, skipped=%d", - summary["status"], - summary["new_urls"], - summary["sold_marked"], - summary["skipped_existing"], + # Circuit breaker: если N подряд hourly-запусков фейлили, пропускаем. + _hourly_streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if _cb_open: + return { + "status": "circuit_breaker_open", + "task_id": task_id, + "hourly_failure_streak": _hourly_streak, + } + _progress_cb = lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, ) - return summary + try: + if hourly_mode == "diff": + logger.info("Hourly mode: running sitemap diff sync instead of full listing traversal") + result = _hourly_sitemap_diff_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + elif hourly_mode == "full_refresh": + logger.info("Hourly mode: running sitemap full refresh of all active vehicles") + result = _hourly_sitemap_full_refresh_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + else: + logger.info("Hourly mode: running sitemap rolling refresh of active vehicles") + result = _hourly_sitemap_rolling_refresh_sync( + redis_client=redis_client, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + progress_callback=_progress_cb, + ) + except SitemapDiscoveryError as exc: + logger.warning( + "Sitemap discovery failed (%s); falling back to listing traversal for hourly sync", + exc, + ) + use_hourly_sitemap_sync = False # fall through to listing traversal below + prefer_sitemap_mainline = False # allow segmented fallback + discovery_mode = "listing" # override so use_segmented check passes + + if use_hourly_sitemap_sync: + try: + redis_client.set(SITEMAP_HOURLY_LAST_COUNT_KEY, str(result.get("discovered_urls", 0))) + except Exception: + logger.warning("Failed to persist hourly sitemap count", exc_info=True) + + # Anti-bot guard: при массовом protection/failed не считаем запуск успешным, + # открываем hourly circuit breaker и уходим в controlled retry по beat. + hourly_failures = len(result.get("failures") or []) + hourly_discovered = int(result.get("discovered_urls") or 0) + hourly_failed = int(result.get("cars_failed") or 0) + hourly_protection = int(result.get("protection_events") or 0) + if hourly_discovered > 0: + fail_ratio = hourly_failed / max(1, hourly_discovered) + protection_ratio = hourly_protection / max(1, hourly_discovered) + anti_bot_suspected = ( + (hourly_protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + _streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered: discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + hourly_discovered, + hourly_failed, + hourly_protection, + fail_ratio, + protection_ratio, + _streak, + ) + return { + "status": "anti_bot_detected", + "task_id": task_id, + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": hourly_discovered, + "cars_failed": hourly_failed, + "protection_events": hourly_protection, + "hourly_failure_streak": _streak, + } + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": result.get("discovered_urls", 0), + "new_urls": result.get("new_urls", 0), + "sold_marked": result.get("sold_marked", 0), + } + logger.info( + "sync_listing_task hourly diff completed: status=%s, new=%d, sold=%d, skipped=%d", + summary["status"], + summary["new_urls"], + summary["sold_marked"], + summary["skipped_existing"], + ) + # Hourly успешно — сбрасываем circuit breaker streak. + _clear_hourly_failure_streak(redis_client) + return summary _clear_followup_pending(redis_client) - heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( - redis_client, - SYNC_LISTING_LOCK_KEY, - owner_token, - lock_ttl, - ) self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) # Определяем сегменты из конфига. @@ -1059,12 +1649,17 @@ def sync_listing_task( if use_segmented and settings.celery.parallel_segments: # Сегменты уже завершённые (для bootstrap resume) пропускаем по Redis SET. already_completed: set[int] = set() + cycle_id: str | None = None if force_bootstrap_full_scan: try: + cycle_id, created_new_cycle = _ensure_segments_cycle(redis_client, len(segments)) raw = redis_client.smembers(SYNC_SEGMENTS_PROGRESS_KEY) or set() already_completed = {int(x) for x in raw if str(x).strip().lstrip("-").isdigit()} + if created_new_cycle: + logger.warning("Started new bootstrap cycle: %s", cycle_id) except Exception: already_completed = set() + cycle_id = None pending = [ (idx, seg) for idx, seg in enumerate(segments) @@ -1076,6 +1671,7 @@ def sync_listing_task( if force_bootstrap_full_scan: _set_full_scan_done(redis_client, True) _clear_sync_checkpoint(redis_client) + _clear_segments_progress_state(redis_client) _clear_bootstrap_failure_streak(redis_client) logger.warning("Parallel segments: nothing to dispatch (all completed)") return { @@ -1087,10 +1683,6 @@ def sync_listing_task( "segments_already_completed": len(already_completed), } - # При первом запуске bootstrap фиксируем total, чтобы знать когда остановиться. - if force_bootstrap_full_scan and not already_completed: - _reset_segments_progress(redis_client, len(segments)) - dispatched = 0 for idx, seg in pending: try: @@ -1110,8 +1702,8 @@ def sync_listing_task( logger.warning("Failed to dispatch segment %d", idx, exc_info=True) logger.warning( - "Parallel segments dispatched: %d/%d (already_completed=%d, bootstrap=%s)", - dispatched, len(segments), len(already_completed), force_bootstrap_full_scan, + "Parallel segments dispatched: %d/%d (already_completed=%d, bootstrap=%s, cycle=%s)", + dispatched, len(segments), len(already_completed), force_bootstrap_full_scan, cycle_id, ) return { "status": "success", @@ -1120,9 +1712,11 @@ def sync_listing_task( "segments_total": len(segments), "segments_dispatched": dispatched, "segments_already_completed": len(already_completed), + "segments_cycle_id": cycle_id, } # --- конец параллельной ветки --- + precomputed_result = None resume_from_segment = 0 if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None: resume_from_segment = max(0, last_completed_segment + 1) @@ -1140,8 +1734,129 @@ def sync_listing_task( resume_from_segment, last_completed_segment, ) + checkpoint_streak, should_skip_from_checkpoint = _register_repeated_bootstrap_checkpoint( + redis_client, + last_completed_segment, + ) + if should_skip_from_checkpoint: + skipped_segment = resume_from_segment + logger.error( + "Checkpoint %d repeated (streak=%d); advancing past segment=%d before resume", + last_completed_segment, + checkpoint_streak, + skipped_segment, + ) + _save_last_completed_segment(redis_client, skipped_segment) + _update_task_progress( + redis_client, + task_id=task_id, + stage="bootstrap_repeat_checkpoint_segment_skipped", + ttl_seconds=progress_ttl, + checkpoint_segment=last_completed_segment, + segment_index=skipped_segment, + checkpoint_streak=checkpoint_streak, + ) + resume_from_segment = skipped_segment + 1 + if resume_from_segment >= len(segments): + precomputed_result = { + "status": "partial_success", + "run_id": None, + "full_scan_completed": True, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 0, + "failures": [{ + "vehicle_url": f"segment_{skipped_segment}", + "error": f"Skipped after repeated bootstrap checkpoint loops on segment {skipped_segment}", + }], + } + else: + logger.warning( + "Bootstrap resume will continue from segment=%d after repeated checkpoint skip of segment=%d", + resume_from_segment, + skipped_segment, + ) + + if ( + force_bootstrap_full_scan + and use_segmented + and stalled_segment is not None + and 0 <= stalled_segment < len(segments) + and stalled_segment >= resume_from_segment + ): + logger.error( + "Skipping previously stalled segment=%d and advancing checkpoint before resume", + stalled_segment, + ) + _save_last_completed_segment(redis_client, stalled_segment) + _clear_stalled_segment(redis_client) + resume_from_segment = stalled_segment + 1 + _update_task_progress( + redis_client, + task_id=task_id, + stage="bootstrap_stalled_segment_skipped", + ttl_seconds=progress_ttl, + segment_index=stalled_segment, + ) + + if force_bootstrap_full_scan and use_segmented and resume_from_segment < len(segments): + resume_streak, should_skip_segment = _register_bootstrap_resume_target( + redis_client, + resume_from_segment, + ) + if should_skip_segment: + skipped_segment = resume_from_segment + logger.error( + "Segment %d is stuck on bootstrap resume (streak=%d); skipping it and advancing checkpoint", + skipped_segment, + resume_streak, + ) + _save_last_completed_segment(redis_client, skipped_segment) + _update_task_progress( + redis_client, + task_id=task_id, + stage="bootstrap_resume_segment_skipped", + ttl_seconds=progress_ttl, + segment_index=skipped_segment, + resume_streak=resume_streak, + ) + resume_from_segment = skipped_segment + 1 + if resume_from_segment >= len(segments): + precomputed_result = { + "status": "partial_success", + "run_id": None, + "full_scan_completed": True, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 0, + "failures": [{ + "vehicle_url": f"segment_{skipped_segment}", + "error": f"Skipped after repeated bootstrap resume loops on segment {skipped_segment}", + }], + } + else: + logger.warning( + "Bootstrap resume will continue from segment=%d after skipping segment=%d", + resume_from_segment, + skipped_segment, + ) + + def _progress_cb_main(stage, meta): + _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + **meta, + ) + def _job(): with IAAIScraper() as scraper: + scraper.set_progress_callback(_progress_cb_main) if use_segmented: return scraper.sync_listing_segmented( segments=segments, @@ -1162,30 +1877,52 @@ def sync_listing_task( only_new=effective_only_new, ) - result = _run_browser_job(_job) + result = precomputed_result if precomputed_result is not None else _run_browser_job(_job) if force_bootstrap_full_scan: bootstrap_completed = bool(result.get("full_scan_completed")) if bootstrap_completed: - _set_full_scan_done(redis_client, True) + _set_full_scan_done(redis_client, False if always_full_scan else True) _clear_sync_checkpoint(redis_client) + _clear_segments_progress_state(redis_client) + _clear_bootstrap_resume_target(redis_client) + _clear_stalled_segment(redis_client) _clear_bootstrap_failure_streak(redis_client) - logger.info("Bootstrap full scan completed; hourly schedule continues") + _clear_bootstrap_continuation_streak(redis_client) + if always_full_scan: + logger.info("Full scan completed; keeping bootstrap mode for next run (always full scan enabled)") + else: + logger.info("Bootstrap full scan completed; hourly schedule continues") else: _set_full_scan_done(redis_client, False) listing_payload = result.get("listing") if isinstance(result.get("listing"), dict) else {} + anti_bot_detected = bool(result.get("anti_bot_detected")) had_progress = any( int(result.get(key) or 0) > 0 for key in ("cars_upserted", "images_upserted", "skipped_existing", "total_discovered") ) or int(listing_payload.get("vehicles_collected") or 0) > 0 - count_as_failure = str(result.get("status") or "") == "failed" and not had_progress - logger.info("Bootstrap full scan not complete yet; queuing immediate continuation") + count_as_failure = anti_bot_detected or (str(result.get("status") or "") == "failed" and not had_progress) + followup_delay = max(3600, int(settings.celery.beat_sync_interval_minutes) * 60) + followup_reason = "bootstrap_not_completed" + if anti_bot_detected: + followup_delay = 180 + followup_reason = "bootstrap_anti_bot_detected" + logger.warning( + "Bootstrap anti-bot guard: protection_events=%s fail_ratio=%s protection_ratio=%s; scheduling delayed continuation", + result.get("protection_events"), + result.get("fail_ratio"), + result.get("protection_ratio"), + ) + else: + logger.info("Bootstrap full scan not complete yet; queuing immediate continuation") _enqueue_bootstrap_followup( - "bootstrap_not_completed", + followup_reason, + delay_seconds=followup_delay, count_as_failure=count_as_failure, ) else: _clear_sync_checkpoint(redis_client) + _clear_bootstrap_resume_target(redis_client) summary = { "task_id": task_id, @@ -1193,11 +1930,37 @@ def sync_listing_task( "status": result.get("status", "success"), "cars_upserted": result.get("cars_upserted", 0), "cars_failed": result.get("cars_failed", 0), + "protection_events": result.get("protection_events", 0), "images_upserted": result.get("images_upserted", 0), "skipped_existing": result.get("skipped_existing", 0), "elapsed_seconds": result.get("elapsed_seconds"), "failures_count": len(result.get("failures") or []), } + if not force_bootstrap_full_scan: + discovered = int(result.get("total_discovered") or result.get("total") or 0) + failed = int(summary["cars_failed"] or 0) + protection = int(summary["protection_events"] or 0) + if discovered > 0: + fail_ratio = failed / max(1, discovered) + protection_ratio = protection / max(1, discovered) + anti_bot_suspected = ( + (protection >= 30 and protection_ratio >= 0.10) + or fail_ratio >= 0.30 + ) + if anti_bot_suspected: + streak = _bump_hourly_failure_streak(redis_client) + logger.error( + "Hourly anti-bot guard triggered (listing fallback): discovered=%d failed=%d protection=%d fail_ratio=%.2f protection_ratio=%.2f streak=%d", + discovered, + failed, + protection, + fail_ratio, + protection_ratio, + streak, + ) + summary["status"] = "anti_bot_detected" + summary["hourly_failure_streak"] = streak + return summary logger.info( "sync_listing_task completed: status=%s, %d upserted, %d failed, failures=%d", summary["status"], @@ -1214,18 +1977,47 @@ def sync_listing_task( if force_bootstrap_full_scan: _set_full_scan_done(redis_client, False) _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=False) - # Partial progress уже записан в БД через finish_sync_run. - # Не retry — следующий запуск продолжит обработку по расписанию. + else: + # Hourly: ставим продолжение, но только если circuit breaker не открыт. + _bump_hourly_failure_streak(redis_client) + _streak, _cb_open = _check_hourly_circuit_breaker(redis_client) + if not _cb_open: + followup_ttl = max(600, int(lock_ttl)) + if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): + try: + self.app.send_task( + "iaai_scraper.worker.tasks.sync_listing_task", + kwargs={ + "make": make, + "model": model, + "lane": lane, + "limit": limit, + "only_new": only_new, + }, + queue="scraping", + countdown=10, + expires=followup_ttl, + ) + logger.info("Queued immediate continuation after soft timeout") + except Exception: + _clear_followup_pending(redis_client) + logger.warning("Failed to queue continuation after soft timeout", exc_info=True) + else: + logger.info("Continuation after soft timeout already pending; skip duplicate enqueue") + else: + logger.warning("Skipping continuation: hourly circuit breaker open (%d failures)", _streak) return { "status": "timed_out", "task_id": task_id, "reason": "soft_time_limit_exceeded", - "note": "partial progress saved to DB; bootstrap continuation queued", + "note": "partial progress saved to DB; continuation queued", } except Exception as exc: logger.error("sync_listing_task failed: %s", exc, exc_info=True) - # Retry только на не-таймаутные ошибки (сеть, БД, браузер). + # Hourly circuit breaker: фиксируем ошибку. + if not force_bootstrap_full_scan: + _bump_hourly_failure_streak(redis_client) try: if force_bootstrap_full_scan: _set_full_scan_done(redis_client, False) @@ -1236,101 +2028,22 @@ def sync_listing_task( if force_bootstrap_full_scan: _set_full_scan_done(redis_client, False) _enqueue_bootstrap_followup("max_retries_exceeded", count_as_failure=True) + # Non-bootstrap: НЕ ставим continuation — beat поставит новую задачу + # через beat_sync_interval_minutes. Бесконечный retry при ошибках + # приводит к молотилке запросов и бану. return { "status": "failed", "task_id": task_id, "error": str(exc), } finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=5) if heartbeat_stop is not None: heartbeat_stop.set() if heartbeat_thread is not None: heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) if lock_acquired: _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) - - -# Новые задачи для ingestion pipeline - -@shared_task( - name="iaai_scraper.worker.tasks.discover_vehicles_task", - bind=True, - max_retries=2, - default_retry_delay=60, - acks_late=True, -) -def discover_vehicles_task(self, max_urls: int | None = None): - """Задача для обнаружения новых URL автомобилей.""" - from ..discovery_service import DiscoveryService - - try: - discovery = DiscoveryService() - count = discovery.discover_new_vehicles(max_urls=max_urls) - logger.info("discover_vehicles_task completed: %d candidates added", count) - return {"status": "success", "candidates_added": count} - except Exception as exc: - logger.error("discover_vehicles_task failed: %s", exc, exc_info=True) - raise self.retry(exc=exc) - - -@shared_task( - name="iaai_scraper.worker.tasks.fetch_pending_candidates_task", - bind=True, - max_retries=2, - default_retry_delay=30, - acks_late=True, -) -def fetch_pending_candidates_task(self, limit: int = 10): - """Задача для захвата данных ожидающих кандидатов.""" - from ..fetch_service import FetchService - - try: - fetch = FetchService() - count = fetch.process_pending_candidates(limit=limit) - logger.info("fetch_pending_candidates_task completed: %d candidates processed", count) - return {"status": "success", "candidates_processed": count} - except Exception as exc: - logger.error("fetch_pending_candidates_task failed: %s", exc, exc_info=True) - raise self.retry(exc=exc) - - -@shared_task( - name="iaai_scraper.worker.tasks.enrich_snapshots_task", - bind=True, - max_retries=2, - default_retry_delay=30, - acks_late=True, -) -def enrich_snapshots_task(self, limit: int = 10): - """Задача для парсинга и обогащения snapshots.""" - from ..enrichment_service import EnrichmentService - - try: - enrichment = EnrichmentService() - count = enrichment.process_unparsed_snapshots(limit=limit) - logger.info("enrich_snapshots_task completed: %d snapshots enriched", count) - return {"status": "success", "snapshots_enriched": count} - except Exception as exc: - logger.error("enrich_snapshots_task failed: %s", exc, exc_info=True) - raise self.retry(exc=exc) - - -@shared_task( - name="iaai_scraper.worker.tasks.run_ingestion_pipeline_task", - bind=True, - max_retries=2, - default_retry_delay=120, - acks_late=True, -) -def run_ingestion_pipeline_task(self): - """Задача для запуска полного ingestion pipeline.""" - from ..scheduler_service import SchedulerService - - try: - scheduler = SchedulerService() - scheduler.run_full_pipeline() - logger.info("run_ingestion_pipeline_task completed") - return {"status": "success"} - except Exception as exc: - logger.error("run_ingestion_pipeline_task failed: %s", exc, exc_info=True) - raise self.retry(exc=exc) diff --git a/pyproject.toml b/pyproject.toml index a183db0..7b5357a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,6 @@ requires-python = ">=3.11" dependencies = [ "alembic>=1.14.0", "celery>=5.4.0", - "curl-cffi>=0.11.0", "fastapi>=0.115.0", "playwright>=1.53.0", "psycopg2-binary>=2.9.9", @@ -21,8 +20,6 @@ dependencies = [ "sqlalchemy>=2.0.32", "uvicorn>=0.34.0", "urllib3>=2.0.0", - "orjson>=3.10.0", - "brotli>=1.1.0", ] [project.optional-dependencies] diff --git a/tests/test_listing.py b/tests/test_listing.py index 3595d79..7d8e991 100644 --- a/tests/test_listing.py +++ b/tests/test_listing.py @@ -8,83 +8,26 @@ from iaai_scraper.browser.listing import ListingCollector class _FakePage: - def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None: + def __init__(self, counts: dict[str, int]) -> None: self._counts = counts - self._attrs = attrs or {} self._evaluate_result = False self._evaluate_values: list[object] = [] - self._html = "" - self.url = "https://www.iaai.com/Vehiclelisting/Cars" - self._current_page_number: int | None = None - self._vehicle_hrefs: list[str] = [] - self._wait_for_function_error: Exception | None = None - self._clicks: dict[str, int] = {} - self._goto_calls: list[str] = [] class _Locator: - def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None: - self._page = page - self._selector = selector + def __init__(self, count_value: int) -> None: self._count_value = count_value - @property - def first(self) -> "_FakePage._Locator": - return self - def count(self) -> int: return self._count_value - def is_visible(self, timeout: int | None = None) -> bool: - _ = timeout - return False + def locator(self, selector: str) -> "_FakePage._Locator": + return _FakePage._Locator(self._counts.get(selector, 0)) - def get_attribute(self, name: str, timeout: int | None = None) -> str | None: - _ = timeout - return self._page._attrs.get(self._selector, {}).get(name) - - def click(self, timeout: int | None = None) -> None: - _ = timeout - self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1 - return None - - class _HtmlLocator: - def __init__(self, html: str) -> None: - self._html = html - - def inner_html(self, timeout: int | None = None) -> str: - _ = timeout - return self._html - - def locator(self, selector: str): - if selector == "html": - return _FakePage._HtmlLocator(self._html) - return _FakePage._Locator(self, selector, self._counts.get(selector, 0)) - - def evaluate(self, script: str, *args): - _ = args + def evaluate(self, _script: str): if self._evaluate_values: return self._evaluate_values.pop(0) - if "querySelectorAll" in script and "VehicleDetail" in script: - return list(self._vehicle_hrefs) - if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script: - return self._current_page_number if self._current_page_number is not None else self._evaluate_result return self._evaluate_result - def wait_for_selector(self, selector: str, timeout: int | None = None) -> None: - _ = selector, timeout - return None - - def wait_for_function(self, script: str, timeout: int | None = None) -> None: - _ = script, timeout - if self._wait_for_function_error is not None: - raise self._wait_for_function_error - return None - - def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None: - _ = wait_until, timeout - self._goto_calls.append(url) - self.url = url - class TestListingUnit(unittest.TestCase): def test_pagination_detection_and_page_number(self) -> None: @@ -120,67 +63,6 @@ class TestListingUnit(unittest.TestCase): ], ) - def test_has_next_page_from_html(self) -> None: - collector = ListingCollector(Settings(), HumanPacer(Settings())) - html = 'Next' - self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42)) - self.assertFalse(collector._has_next_page_from_html("
done
", current_page_number=42)) - - def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None: - collector = ListingCollector(Settings(), HumanPacer(Settings())) - - url = collector._build_listing_page_url( - "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar", - 44, - ) - - self.assertEqual( - url, - "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44", - ) - - def test_collect_current_page_uses_html_first(self) -> None: - collector = ListingCollector(Settings(), HumanPacer(Settings())) - page = _FakePage({}) - page._html = 'Car 555Next' - - result = collector.collect_current_page(page, page_number=1) - - self.assertEqual(len(result.vehicle_links), 1) - self.assertEqual(result.vehicle_links[0].lot_number, "555") - self.assertTrue(result.next_page_detected) - - def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None: - page = _FakePage({}) - page._wait_for_function_error = RuntimeError("same content") - page._current_page_number = 41 - page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"] - - self.assertFalse( - ListingCollector._wait_for_navigation_result( - page, - "/VehicleDetail/111~US", - 41, - old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"), - ) - ) - - def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None: - collector = ListingCollector(Settings(), HumanPacer(Settings())) - page = _FakePage( - {ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1}, - attrs={ - "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"}, - }, - ) - page._wait_for_function_error = RuntimeError("same content") - page._current_page_number = 41 - page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"] - - self.assertFalse(collector.go_to_next_page(page, expected_page_number=41)) - self.assertGreaterEqual(len(page._goto_calls), 1) - self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_resilience.py b/tests/test_resilience.py new file mode 100644 index 0000000..8b2bf81 --- /dev/null +++ b/tests/test_resilience.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from iaai_scraper.scraper import IAAIScraper +from iaai_scraper.core.config import Settings +from iaai_scraper.worker import tasks + + +class TestResilience(unittest.TestCase): + def _make_scraper(self) -> IAAIScraper: + s = Settings() + s.log_level = "CRITICAL" + s.database.url = "sqlite://" + return IAAIScraper(s) + + def test_update_task_progress_updates_global_marker(self) -> None: + redis_client = MagicMock() + pipe = MagicMock() + redis_client.pipeline.return_value = pipe + + tasks._update_task_progress( + redis_client, + task_id="task-abc", + stage="batch_upserted", + ttl_seconds=180, + cars_upserted=10, + ) + + redis_client.pipeline.assert_called_once() + self.assertEqual(pipe.set.call_count, 2) + first_call = pipe.set.call_args_list[0] + second_call = pipe.set.call_args_list[1] + + self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc")) + self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY) + pipe.execute.assert_called_once() + + def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None: + scraper = self._make_scraper() + scraper.settings.celery.batch_size = 50 + + page = MagicMock() + empty_page_result = SimpleNamespace( + page_number=1, + vehicle_links=[], + next_page_detected=True, + ) + + scraper._open_listing_for_stream = MagicMock(return_value=( + page, + {"make": None, "model": None, "year_min": None, "year_max": None}, + )) + scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result) + scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, [])) + scraper.sync_batch = MagicMock() + + result = scraper._sync_listing_streaming( + make=None, + model=None, + lane="iaai_cars", + limit=None, + effective_only_new=False, + started_at=0.0, + listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST", + ) + + self.assertEqual(result["total"], 0) + self.assertEqual(result["cars_upserted"], 0) + self.assertEqual(result["cars_failed"], 0) + self.assertEqual(result["listing"]["pages_collected"], 1) + scraper._recover_empty_listing_page.assert_called_once() + scraper.sync_batch.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 70b8d4e..7d6192a 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -1,13 +1,12 @@ from __future__ import annotations import unittest +from concurrent.futures import TimeoutError as FuturesTimeoutError from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from iaai_scraper.core.config import Settings from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError -from iaai_scraper.core.exceptions import ListingResumeError -from iaai_scraper.discovery import SitemapDiscoveryError from iaai_scraper.scraper import IAAIScraper from iaai_scraper.storage.schemas import CarRecord @@ -87,51 +86,6 @@ class TestScraperSync(unittest.TestCase): scraper2._sync_listing_streaming.assert_called_once() scraper2.collect_listing.assert_not_called() - def test_full_scan_uses_sitemap_discovery_path(self) -> None: - scraper = self._make_scraper() - scraper.settings.discovery.mode = "listing" - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=88) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0) - scraper._sync_listing_via_sitemap = MagicMock(return_value={ - "listing": { - "vehicles_collected": 123, - "early_stopped": False, - "truncated_by_time_budget": False, - "pagination_interrupted": False, - "source": "sitemap", - }, - "total": 123, - "skipped_existing": 0, - "cars_upserted": 120, - "cars_failed": 3, - "images_upserted": 500, - "failures": [{"vehicle_url": "v", "error": "e"}], - "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"}, - }) - scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used")) - - result = scraper.sync_listing() - - scraper._sync_listing_via_sitemap.assert_called_once() - scraper._sync_listing_streaming.assert_not_called() - self.assertEqual(result["cars_upserted"], 120) - self.assertTrue(result["full_scan_completed"]) - - def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None: - scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=89) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0) - scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down")) - result = scraper.sync_listing() - - scraper._sync_listing_via_sitemap.assert_called_once() - self.assertEqual(result["status"], "failed") - self.assertEqual(result["cars_upserted"], 0) - def test_segmented_sync_calls_per_segment_and_resumes(self) -> None: scraper = self._make_scraper() scraper.persistence.create_tables = MagicMock() @@ -315,125 +269,50 @@ class TestScraperSync(unittest.TestCase): self.assertEqual(result["cars_upserted"], 1) self.assertEqual(result["total"], 1) - def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None: + def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None: scraper = self._make_scraper() - scraper.settings.celery.batch_size = 1000 - - page = MagicMock() - page_result = SimpleNamespace( - page_number=1, - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")], - next_page_detected=True, - ) - - scraper._get_page_with_warmup = MagicMock(return_value=page) - scraper.listing_collector.open_cars_listing = MagicMock() - scraper.listing_collector.apply_filters = MagicMock(return_value={ - "make": None, - "model": None, - "year_min": None, - "year_max": None, - }) - scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) - scraper.listing_collector.go_to_next_page = MagicMock(return_value=False) - scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"]) - scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed")) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 0, - "cars_failed": 0, + scraper.persistence.upsert_cars_batch = MagicMock(return_value={ + "inserted": 1, + "updated": 0, "images_upserted": 0, - "failures": [], }) - result = scraper._sync_listing_streaming( - make=None, - model=None, - lane="iaai_cars", - limit=None, - effective_only_new=False, - started_at=0.0, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", - ) + urls = [ + "https://www.iaai.com/VehicleDetail/111~US", + "https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/VehicleDetail/333~US", + ] + first_record = CarRecord.model_validate(make_db_record("111")) - self.assertTrue(result["listing"]["pagination_interrupted"]) + class _FakeExecutor: + def __init__(self, *args, **kwargs): + pass - def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None: - scraper = self._make_scraper() - scraper.settings.celery.batch_size = 1000 + def __enter__(self): + return self - page = MagicMock() - first_page = SimpleNamespace( - page_number=1, - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")], - next_page_detected=False, - ) - second_page = SimpleNamespace( - page_number=2, - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")], - next_page_detected=False, - ) + def __exit__(self, exc_type, exc, tb): + return False - scraper._get_page_with_warmup = MagicMock(return_value=page) - scraper.listing_collector.open_cars_listing = MagicMock() - scraper.listing_collector.apply_filters = MagicMock(return_value={ - "make": None, - "model": None, - "year_min": None, - "year_max": None, - }) - scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page]) - scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False]) - scraper._extract_page_urls = MagicMock(side_effect=[ - ["https://www.iaai.com/VehicleDetail/111~US"], - ["https://www.iaai.com/VehicleDetail/222~US"], - ]) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 2, - "cars_failed": 0, - "images_upserted": 0, - "failures": [], - }) + def map(self, fn, iterable, timeout=None): # noqa: ARG002 + yield 0, first_record + raise FuturesTimeoutError() - result = scraper._sync_listing_streaming( - make=None, - model=None, - lane="iaai_cars", - limit=None, - effective_only_new=False, - started_at=0.0, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", - ) + with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ + patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ + patch.object(scraper, "_browser_fallback_parallel", return_value={ + "records": [], + "failures": [], + "cars_failed": 0, + "protection_events": 0, + }) as fallback_mock: + result = scraper.sync_batch(urls) - self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2) - scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2) - self.assertEqual(result["listing"]["pages_collected"], 2) - - def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None: - scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=77) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock() - scraper._sync_listing_streaming = MagicMock(return_value={ - "listing": { - "vehicles_collected": 10, - "early_stopped": False, - "truncated_by_time_budget": False, - "pagination_interrupted": True, - }, - "total": 10, - "skipped_existing": 0, - "cars_upserted": 10, - "cars_failed": 0, - "images_upserted": 0, - "failures": [], - "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"}, - }) - - result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA") - - self.assertTrue(result["full_scan_completed"] is False) - scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called() + self.assertEqual(result["cars_upserted"], 1) + fallback_urls = fallback_mock.call_args.args[0] + self.assertEqual(len(fallback_urls), 2) + self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]}) + self.assertNotIn(urls[0], {u for u, _ in fallback_urls}) if __name__ == "__main__": diff --git a/tests/test_self_heal.py b/tests/test_self_heal.py new file mode 100644 index 0000000..f5c74e1 --- /dev/null +++ b/tests/test_self_heal.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock, patch + +from iaai_scraper.worker import self_heal + + +class TestSelfHeal(unittest.TestCase): + def test_read_last_progress_ts_uses_global_key(self) -> None: + redis_client = MagicMock() + redis_client.get.return_value = "1776800000" + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertEqual(ts, 1776800000) + redis_client.scan_iter.assert_not_called() + + def test_read_last_progress_ts_fallbacks_to_task_progress_keys(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + self_heal.GLOBAL_PROGRESS_TS_KEY: None, + "iaai:state:task_progress:a": '{"ts": 100}', + "iaai:state:task_progress:b": '{"ts": 250}', + "iaai:state:task_progress:c": '{"ts": 150}', + }.get(key) + redis_client.scan_iter.return_value = [ + "iaai:state:task_progress:a", + "iaai:state:task_progress:b", + "iaai:state:task_progress:c", + ] + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertEqual(ts, 250) + + def test_read_last_progress_ts_ignores_broken_payloads(self) -> None: + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: { + self_heal.GLOBAL_PROGRESS_TS_KEY: None, + "iaai:state:task_progress:a": "{bad-json}", + "iaai:state:task_progress:b": '{"foo": "bar"}', + }.get(key) + redis_client.scan_iter.return_value = [ + "iaai:state:task_progress:a", + "iaai:state:task_progress:b", + ] + + ts = self_heal._read_last_progress_ts(redis_client) + + self.assertIsNone(ts) + + @patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("iaai_scraper.worker.self_heal.os.kill") + @patch("builtins.open") + def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None: + open_mock.return_value.__enter__.return_value.read.return_value = "123" + # SIGTERM -> process alive check (pid,0) -> SIGKILL + kill_mock.side_effect = [None, None, None] + + self_heal._kill_worker_process() + + self.assertEqual(kill_mock.call_args_list[0].args[0], 123) + self.assertEqual(kill_mock.call_args_list[1].args, (123, 0)) + self.assertEqual(kill_mock.call_args_list[2].args[0], 123) + + @patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("iaai_scraper.worker.self_heal.os.kill") + @patch("builtins.open") + def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None: + open_mock.return_value.__enter__.return_value.read.return_value = "123" + kill_mock.side_effect = [None, ProcessLookupError()] + + self_heal._kill_worker_process() + + # Только SIGTERM и проверка существования процесса. + self.assertEqual(len(kill_mock.call_args_list), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sitemap_discovery.py b/tests/test_sitemap_discovery.py deleted file mode 100644 index c6f3a2e..0000000 --- a/tests/test_sitemap_discovery.py +++ /dev/null @@ -1,217 +0,0 @@ -from __future__ import annotations - -import unittest -from unittest.mock import patch - -from iaai_scraper.core.config import Settings -from iaai_scraper.discovery.sitemap import ( - SitemapBlockedError, - SitemapDiscoveryError, - SitemapFetchResult, - _filter_vehicle_urls, - discover_vehicle_urls_from_sitemap, - discover_vehicle_urls_from_sitemap_with_stats, -) - - -class TestSitemapDiscovery(unittest.TestCase): - @staticmethod - def _fetch_result(url: str, payload: bytes, *, source: str = "curl_cffi") -> SitemapFetchResult: - return SitemapFetchResult(url=url, payload=payload, source=source) - - def test_filter_vehicle_urls_dedupes_and_normalizes(self) -> None: - urls = _filter_vehicle_urls([ - "https://www.iaai.com/VehicleDetail/111~US?foo=1", - "https://www.iaai.com/VehicleDetail/111~US?bar=2", - "https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/about", - ]) - - self.assertEqual( - urls, - [ - "https://www.iaai.com/VehicleDetail/111~US", - "https://www.iaai.com/VehicleDetail/222~US", - ], - ) - - def test_discover_vehicle_urls_from_sitemap(self) -> None: - index_xml = b""" - - https://www.iaai.com/sitemap-a.xml - https://www.iaai.com/sitemap-b.xml - https://www.iaai.com/sitemapauctions1.xml - - """ - sitemap_a = b""" - - https://www.iaai.com/VehicleDetail/111~US - https://www.iaai.com/VehicleDetail/222~US?x=1 - - """ - sitemap_b = b""" - - https://www.iaai.com/VehicleDetail/222~US?y=2 - https://www.iaai.com/VehicleDetail/333~US - - """ - - def _fake_fetch(url: str) -> SitemapFetchResult: - if url.endswith("sitemap_index.xml"): - return self._fetch_result(url, index_xml) - if url.endswith("sitemap-a.xml"): - return self._fetch_result(url, sitemap_a) - if url.endswith("sitemap-b.xml"): - return self._fetch_result(url, sitemap_b) - raise AssertionError(f"unexpected url: {url}") - - with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): - result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml") - - self.assertEqual( - result, - [ - "https://www.iaai.com/VehicleDetail/111~US", - "https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/VehicleDetail/333~US", - ], - ) - - def test_discover_vehicle_urls_raises_on_empty_index(self) -> None: - empty_index = b"" - with patch( - "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", - return_value=self._fetch_result("https://www.iaai.com/sitemap_index.xml", empty_index), - ): - with self.assertRaises(SitemapDiscoveryError): - discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml") - - def test_discover_vehicle_urls_skips_malformed_and_non_vehicle_sitemaps(self) -> None: - sitemap_vehicle = b""" - - https://www.iaai.com/VehicleDetail/111~US - - """ - sitemap_non_vehicle = b""" - - https://www.iaai.com/SalesList/111~US/04222026 - - """ - index_xml = b""" - - https://www.iaai.com/sitemap1.xml - https://www.iaai.com/sitemapbranches1.xml - https://www.iaai.com/sitemap2.xml - https://www.iaai.com/sitemap3.xml - - """ - - def _fake_fetch(url: str) -> SitemapFetchResult: - if url.endswith("sitemap_index.xml"): - return self._fetch_result(url, index_xml) - if url.endswith("sitemap1.xml"): - return self._fetch_result(url, sitemap_vehicle) - if url.endswith("sitemap2.xml"): - return self._fetch_result(url, sitemap_non_vehicle) - if url.endswith("sitemap3.xml"): - raise SitemapDiscoveryError("broken sitemap") - raise AssertionError(f"unexpected url: {url}") - - with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): - result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml") - - self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"]) - - def test_discover_vehicle_urls_falls_back_to_regex_loc_extraction(self) -> None: - malformed_index = ( - b"" - b"https://www.iaai.com/sitemap1.xml" - b" - https://www.iaai.com/VehicleDetail/111~US - - """ - - def _fake_fetch(url: str) -> SitemapFetchResult: - if url.endswith("sitemap_index.xml"): - return self._fetch_result(url, malformed_index) - if url.endswith("sitemap1.xml"): - return self._fetch_result(url, sitemap_vehicle) - raise AssertionError(f"unexpected url: {url}") - - with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): - result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml") - - self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"]) - - def test_discovery_uses_direct_probe_when_index_has_no_urls(self) -> None: - index_xml = b"" - sitemap_one = b""" - - https://www.iaai.com/VehicleDetail/111~US - - """ - - settings = Settings() - settings.discovery.sitemap_direct_probe_limit = 2 - settings.discovery.sitemap_direct_probe_stop_after_misses = 1 - - def _fake_fetch(url: str) -> SitemapFetchResult: - if url.endswith("sitemap_index.xml"): - return self._fetch_result(url, index_xml) - if url.endswith("sitemap1.xml"): - return self._fetch_result(url, sitemap_one) - raise SitemapDiscoveryError("not found") - - with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): - result = discover_vehicle_urls_from_sitemap_with_stats( - "https://www.iaai.com/sitemap_index.xml", - settings=settings, - ) - - self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"]) - self.assertEqual(result.stats.direct_probe_hits, 1) - - def test_discovery_stats_report_direct_probe_hits(self) -> None: - index_xml = b"" - sitemap1 = b""" - - https://www.iaai.com/VehicleDetail/111~US - - """ - - def _fake_fetch(url: str): - if url.endswith("sitemap_index.xml"): - return self._fetch_result(url, index_xml) - if url.endswith("sitemap1.xml"): - return self._fetch_result(url, sitemap1) - raise SitemapDiscoveryError("not found") - - with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): - result = discover_vehicle_urls_from_sitemap_with_stats("https://www.iaai.com/sitemap_index.xml") - - self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"]) - self.assertEqual(result.stats.transport, "curl_cffi") - self.assertEqual(result.stats.direct_probe_hits, 1) - self.assertGreaterEqual(result.stats.direct_probe_misses, 1) - - def test_block_page_raises_discovery_error(self) -> None: - settings = Settings() - settings.discovery.sitemap_direct_probe_limit = 1 - settings.discovery.sitemap_direct_probe_stop_after_misses = 1 - - with patch( - "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", - side_effect=SitemapBlockedError("Anti-bot page returned for https://www.iaai.com/sitemap_index.xml"), - ): - with self.assertRaises(SitemapDiscoveryError): - discover_vehicle_urls_from_sitemap( - "https://www.iaai.com/sitemap_index.xml", - settings=settings, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py index 378d000..32ac019 100644 --- a/tests/test_worker_tasks.py +++ b/tests/test_worker_tasks.py @@ -1,215 +1,15 @@ from __future__ import annotations +import json +from dataclasses import replace import unittest from unittest.mock import MagicMock, patch from iaai_scraper.worker import tasks - - -def make_settings_stub(*, parallel_segments: bool = False) -> MagicMock: - settings_stub = MagicMock() - settings_stub.celery.parallel_segments = parallel_segments - settings_stub.celery.task_soft_time_limit = 3300 - settings_stub.celery.task_time_limit = 3600 - settings_stub.celery.task_stall_timeout_seconds = 600 - settings_stub.listing.listing_segments_json = "ignored" - settings_stub.discovery.mode = "listing" - settings_stub.discovery.hourly_mode = "rolling_refresh" - return settings_stub +from iaai_scraper.core.config import settings as base_settings class TestWorkerTaskLockHelpers(unittest.TestCase): - def test_sync_listing_task_uses_hourly_sitemap_even_when_mode_is_not_sitemap(self) -> None: - settings_stub = make_settings_stub(parallel_segments=False) - settings_stub.discovery.mode = "listing" - settings_stub.discovery.hourly_mode = "rolling_refresh" - - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "Settings", return_value=settings_stub), \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={ - "status": "success", - "run_id": None, - "cars_upserted": 25, - "cars_failed": 0, - "images_upserted": 50, - "skipped_existing": 100, - "elapsed_seconds": None, - "failures": [], - "hourly_mode": "sitemap_rolling_refresh", - "discovered_urls": 105, - "new_urls": 5, - "refresh_urls": 20, - "sold_marked": 2, - }) as hourly_sync, \ - patch.object(tasks.sync_listing_task, "update_state"): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - tasks.sync_listing_task.push_request(id="task-hourly-force-sitemap") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh") - hourly_sync.assert_called_once() - release_lock.assert_called_once() - - def test_sync_listing_task_uses_hourly_sitemap_rolling_refresh_after_bootstrap(self) -> None: - settings_stub = make_settings_stub(parallel_segments=False) - settings_stub.discovery.mode = "sitemap" - settings_stub.discovery.hourly_mode = "rolling_refresh" - - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "Settings", return_value=settings_stub), \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={ - "status": "success", - "run_id": None, - "cars_upserted": 25, - "cars_failed": 0, - "images_upserted": 50, - "skipped_existing": 100, - "elapsed_seconds": None, - "failures": [], - "hourly_mode": "sitemap_rolling_refresh", - "discovered_urls": 105, - "new_urls": 5, - "refresh_urls": 20, - "sold_marked": 2, - }) as hourly_sync, \ - patch.object(tasks.sync_listing_task, "update_state"): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - tasks.sync_listing_task.push_request(id="task-hourly") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh") - hourly_sync.assert_called_once() - release_lock.assert_called_once() - - def test_sync_listing_task_can_use_hourly_sitemap_diff_when_configured(self) -> None: - settings_stub = make_settings_stub(parallel_segments=False) - settings_stub.discovery.mode = "sitemap" - settings_stub.discovery.hourly_mode = "diff" - - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "Settings", return_value=settings_stub), \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=True), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_hourly_sitemap_diff_sync", return_value={ - "status": "success", - "run_id": None, - "cars_upserted": 5, - "cars_failed": 0, - "images_upserted": 10, - "skipped_existing": 100, - "elapsed_seconds": None, - "failures": [], - "hourly_mode": "sitemap_diff", - "discovered_urls": 105, - "new_urls": 5, - "sold_marked": 2, - }) as hourly_sync, \ - patch.object(tasks.sync_listing_task, "update_state"): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - tasks.sync_listing_task.push_request(id="task-hourly-diff") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - self.assertEqual(result["hourly_mode"], "sitemap_diff") - hourly_sync.assert_called_once() - release_lock.assert_called_once() - - def test_sync_listing_task_forces_sitemap_full_scan_in_bootstrap(self) -> None: - settings_stub = make_settings_stub(parallel_segments=False) - settings_stub.discovery.mode = "listing" - - with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "Settings", return_value=settings_stub), \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=False), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[{"make": "HONDA"}]): - get_persistence.return_value = MagicMock() - redis_client = MagicMock() - redis_client.get.return_value = None - get_redis.return_value = redis_client - start_heartbeat.return_value = (MagicMock(), MagicMock()) - - sync_listing_mock = MagicMock(return_value={ - "run_id": 99, - "status": "success", - "full_scan_completed": True, - "cars_upserted": 10, - "cars_failed": 0, - "images_upserted": 20, - "skipped_existing": 0, - "elapsed_seconds": 1.0, - "failures": [], - }) - sync_listing_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used")) - scraper = MagicMock() - scraper.sync_listing = sync_listing_mock - scraper.sync_listing_segmented = sync_listing_segmented_mock - scraper_ctx = MagicMock() - scraper_ctx.__enter__.return_value = scraper - scraper_ctx.__exit__.return_value = None - - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): - tasks.sync_listing_task.push_request(id="task-bootstrap-sitemap") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() - - self.assertEqual(result["status"], "success") - sync_listing_mock.assert_called_once_with( - make=None, - model=None, - lane="iaai_cars", - limit=None, - only_new=False, - ) - sync_listing_segmented_mock.assert_not_called() - release_lock.assert_called_once() - def test_lock_acquire_refresh_release(self) -> None: redis_client = MagicMock() @@ -378,17 +178,15 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): self.assertIsNone(tasks._load_last_completed_segment(redis_client)) redis_client.delete.assert_not_called() - def test_sync_listing_bootstrap_prefers_sitemap_over_segment_resume(self) -> None: + def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None: segments = [ {"make": "ACURA"}, {"make": "AUDI"}, {"make": "BMW"}, {"make": "EAGLE"}, ] - settings_stub = make_settings_stub() with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "Settings", return_value=settings_stub), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ @@ -404,15 +202,13 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) - sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used")) - sync_listing_mock = MagicMock(return_value={ + sync_segmented_mock = MagicMock(return_value={ "run_id": 11, "status": "success", "full_scan_completed": True, "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], }) scraper_ctx = MagicMock() scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock scraper_ctx.__exit__.return_value = None with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): @@ -423,42 +219,21 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") - sync_listing_mock.assert_called_once_with( - make=None, - model=None, - lane="iaai_cars", - limit=None, - only_new=False, - ) - sync_segmented_mock.assert_not_called() + # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW). + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2) + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) release_lock.assert_called_once() - def test_sync_listing_hourly_path_ignores_checkpoint_after_full_scan_completed(self) -> None: + def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None: segments = [{"make": "ACURA"}, {"make": "AUDI"}] - settings_stub = make_settings_stub() with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "Settings", return_value=settings_stub), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ - patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={ - "status": "success", - "run_id": None, - "cars_upserted": 1, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "elapsed_seconds": None, - "failures": [], - "hourly_mode": "sitemap_rolling_refresh", - "discovered_urls": 10, - "new_urls": 1, - "refresh_urls": 1, - "sold_marked": 0, - }) as hourly_sync, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() @@ -470,24 +245,32 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) - tasks.sync_listing_task.push_request(id="task-792") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() + sync_segmented_mock = MagicMock(return_value={ + "run_id": 14, "status": "success", "full_scan_completed": True, + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-792") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") - self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh") - hourly_sync.assert_called_once() + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) + self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"]) clear_checkpoint.assert_called() release_lock.assert_called_once() - def test_sync_listing_bootstrap_ignores_beyond_segment_checkpoint(self) -> None: + def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None: segments = [{"make": "ACURA"}, {"make": "AUDI"}] - settings_stub = make_settings_stub() with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "Settings", return_value=settings_stub), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ @@ -503,15 +286,13 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) - sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used")) - sync_listing_mock = MagicMock(return_value={ + sync_segmented_mock = MagicMock(return_value={ "run_id": 15, "status": "success", "full_scan_completed": True, "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], }) scraper_ctx = MagicMock() scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock scraper_ctx.__exit__.return_value = None with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): @@ -522,14 +303,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") - sync_listing_mock.assert_called_once_with( - make=None, - model=None, - lane="iaai_cars", - limit=None, - only_new=False, - ) - sync_segmented_mock.assert_not_called() + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) release_lock.assert_called_once() def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None: @@ -606,38 +380,28 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): self.assertTrue(should_enqueue) def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None: - settings_stub = make_settings_stub(parallel_segments=False) with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "Settings", return_value=settings_stub), \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=False), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_try_set_followup_pending", return_value=True), \ - patch.object( - tasks, - "_bump_bootstrap_failure_streak", - return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False), - ) as bump_streak, \ - patch.object(tasks, "_clear_followup_pending") as clear_pending, \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch.object( - tasks, - "_run_browser_job", - return_value={ - "run_id": 99, - "status": "failed", - "full_scan_completed": False, - "cars_upserted": 0, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "elapsed_seconds": 1.0, - "failures": [{"vehicle_url": "listing", "error": "bad resume"}], - "listing": {"vehicles_collected": 0}, - }, - ): + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_try_set_followup_pending", return_value=True), \ + patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \ + patch.object(tasks, "_clear_followup_pending") as clear_pending, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 99, + "status": "failed", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [{"vehicle_url": "listing", "error": "bad resume"}], + "listing": {"vehicles_collected": 0}, + }): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None @@ -656,6 +420,177 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): clear_pending.assert_called() task_app.send_task.assert_not_called() + def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ + patch.object(tasks, "_release_lock_if_owner"), \ + patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \ + patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-soft-timeout") + try: + result = tasks.sync_listing_task.run(make="Toyota") + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "timed_out") + set_pending.assert_called_once() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ + patch.object(tasks, "_release_lock_if_owner"), \ + patch.object(tasks, "_try_set_followup_pending", return_value=True), \ + patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \ + patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ + patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job", return_value={ + "run_id": 101, + "status": "partial_success", + "full_scan_completed": False, + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 1, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + "listing": {"vehicles_collected": 2}, + }): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: + tasks.sync_listing_task.push_request(id="task-breaker-stop") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "partial_success") + # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly. + self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list)) + self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list)) + clear_checkpoint.assert_called_once() + task_app.send_task.assert_not_called() + + def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None: + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object(tasks, "_run_browser_job") as run_job, \ + patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + run_job.return_value = { + "run_id": 17, + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + } + + tasks.sync_listing_task.push_request(id="task-always-full") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + set_full_scan_done.assert_called_with(redis_client, False) + release_lock.assert_called_once() + + def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.side_effect = lambda key: ( + "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None + ) + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_segmented_mock = MagicMock(return_value={ + "run_id": 18, + "status": "success", + "full_scan_completed": True, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + }) + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__enter__.return_value.sync_listing = MagicMock() + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-always-full-resume") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1) + self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) + release_lock.assert_called_once() + if __name__ == "__main__": unittest.main() \ No newline at end of file