diff --git a/.gitignore b/.gitignore index ad2cc69..05ffa52 100644 --- a/.gitignore +++ b/.gitignore @@ -21,10 +21,10 @@ celerybeat-schedule* tokens_data/ tokens.json -# Ad-hoc debug scripts and one-off downloads -_*.sh -_*.tar.gz -burst_*.ps1 -burst_*.txt -sitemap*.xml -vd_*.html \ No newline at end of file +# Local deployment/debug artifacts +/*.tgz +/*.tar.gz +/*.zip +/NOW() +/_speed_check.sql +/.env.bak* \ No newline at end of file diff --git a/README.md b/README.md index 14a25db..c3d93ec 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,32 @@ docker compose up -d `entrypoint.sh` поднимает SOCKS5→HTTP proxy bridge (если задан `SOCKS5_PROXY_HOST`) и запускает `Xvfb` только для `worker` и CLI scraping-команд. -По умолчанию `beat` запускает сбор листинга **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`). +По умолчанию `beat` запускает синхронизацию **раз в 1 час** (`CELERY_BEAT_SYNC_INTERVAL_MINUTES=60`). + +Текущая стратегия устойчива к багам пагинации IAAI: + +- **первый запуск** делает полный bootstrap через `sitemap`; +- если bootstrap не завершился, автоматически продолжается с чекпоинта; +- **после первого полного прохода** каждый час выполняется проход по `sitemap`. + По умолчанию включён режим **rolling refresh**: + - берутся все URL из sitemap, + - новые URL добавляются сразу, + - исчезнувшие URL помечаются как `is_sold=true`, + - уже записанные авто обновляются **батчами по кругу**, а не все разом. + + Это даёт более стабильную нагрузку и снижает риск не уложиться в час. + + Доступные режимы: + - `IAAI_HOURLY_MODE=rolling_refresh` — рекомендуемый продовый режим; + - `IAAI_HOURLY_MODE=full_refresh` — перепарсить все активные авто за один hourly цикл; + - `IAAI_HOURLY_MODE=diff` — только новые авто + sold. + +За счёт этого система: + +- не зависит от page 39 / 70 / 100, +- не уходит в бесконечный цикл пагинации, +- даёт стабильный hourly refresh уже записанных авто без монолитного полного hourly прохода, +- остаётся быстрой и устойчивой при hourly sync. Swagger-документация: `http://localhost:8000/docs` diff --git a/alembic/versions/004_add_ingestion_tables.py b/alembic/versions/004_add_ingestion_tables.py new file mode 100644 index 0000000..42ec7e1 --- /dev/null +++ b/alembic/versions/004_add_ingestion_tables.py @@ -0,0 +1,81 @@ +# Добавление таблиц для новой архитектуры ingestion: candidates, raw snapshots, parse results, retry queue +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "004_add_ingestion_tables" +down_revision: Union[str, None] = "003_add_composite_indexes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Таблица vehicle_candidates + op.create_table( + "vehicle_candidates", + sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True), + sa.Column("url", sa.String(), nullable=False, unique=True, index=True), + sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True), + sa.Column("status", sa.String(20), nullable=False, server_default="pending", index=True), + sa.Column("priority", sa.Integer(), nullable=False, server_default="0"), + sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), + ) + + # Индексы для vehicle_candidates + op.create_index("ix_vehicle_candidates_url", "vehicle_candidates", ["url"]) + op.create_index("ix_vehicle_candidates_status_discovered", "vehicle_candidates", ["status", "discovered_at"]) + + # Таблица vehicle_raw_snapshots + op.create_table( + "vehicle_raw_snapshots", + sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True), + sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True), + sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True), + sa.Column("method", sa.String(20), nullable=False), + sa.Column("success", sa.Boolean(), nullable=False), + sa.Column("raw_data", sa.Text(), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + ) + + # Индексы для vehicle_raw_snapshots + op.create_index("ix_vehicle_raw_snapshots_candidate_id", "vehicle_raw_snapshots", ["candidate_id"]) + op.create_index("ix_vehicle_raw_snapshots_captured_at", "vehicle_raw_snapshots", ["captured_at"]) + + # Таблица vehicle_parse_results + op.create_table( + "vehicle_parse_results", + sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True), + sa.Column("snapshot_id", sa.Integer(), sa.ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True), + sa.Column("parsed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now(), index=True), + sa.Column("success", sa.Boolean(), nullable=False), + sa.Column("parsed_data", sa.Text(), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + ) + + # Индексы для vehicle_parse_results + op.create_index("ix_vehicle_parse_results_snapshot_id", "vehicle_parse_results", ["snapshot_id"]) + op.create_index("ix_vehicle_parse_results_parsed_at", "vehicle_parse_results", ["parsed_at"]) + + # Таблица vehicle_retry_queue + op.create_table( + "vehicle_retry_queue", + sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True), + sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True), + sa.Column("reason", sa.String(50), nullable=False), + sa.Column("retry_at", sa.DateTime(timezone=True), nullable=False, index=True), + sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), + sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"), + ) + + # Индексы для vehicle_retry_queue + op.create_index("ix_vehicle_retry_queue_candidate_id", "vehicle_retry_queue", ["candidate_id"]) + op.create_index("ix_vehicle_retry_queue_retry_at", "vehicle_retry_queue", ["retry_at"]) + + +def downgrade() -> None: + op.drop_table("vehicle_retry_queue") + op.drop_table("vehicle_parse_results") + op.drop_table("vehicle_raw_snapshots") + op.drop_table("vehicle_candidates") \ No newline at end of file diff --git a/deploy_vps.sh b/deploy_vps.sh new file mode 100644 index 0000000..d9af710 --- /dev/null +++ b/deploy_vps.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +APP_DIR="/opt/iaai_scraper_project" +REPO_URL="${1:-}" + +if [[ -z "${REPO_URL}" ]]; then + echo "Usage: ./deploy_vps.sh " + exit 1 +fi + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + gnupg \ + lsb-release + +install -m 0755 -d /etc/apt/keyrings +if [[ ! -f /etc/apt/keyrings/docker.gpg ]]; then + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg +fi + +chmod a+r /etc/apt/keyrings/docker.gpg +ARCH="$(dpkg --print-architecture)" +CODENAME="$(. /etc/os-release && echo "$VERSION_CODENAME")" +echo \ + "deb [arch=${ARCH} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${CODENAME} stable" \ + > /etc/apt/sources.list.d/docker.list + +apt-get update +apt-get install -y --no-install-recommends \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin + +mkdir -p /opt + +if [[ -d "${APP_DIR}/.git" ]]; then + git -C "${APP_DIR}" fetch --all --prune + git -C "${APP_DIR}" reset --hard origin/HEAD +else + rm -rf "${APP_DIR}" + git clone "${REPO_URL}" "${APP_DIR}" +fi + +cd "${APP_DIR}" + +if [[ ! -f .env ]]; then + cp .env.vps.example .env +fi + +docker compose down --remove-orphans || true +docker compose up -d --build postgres redis migrate api worker beat +docker compose ps diff --git a/iaai_scraper/browser/listing.py b/iaai_scraper/browser/listing.py index 324b25f..d2f56d7 100644 --- a/iaai_scraper/browser/listing.py +++ b/iaai_scraper/browser/listing.py @@ -3,13 +3,12 @@ import re import time from dataclasses import asdict, dataclass, field from typing import Any -from urllib.parse import urljoin +from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse from playwright.sync_api import Page from .pace import HumanPacer from ..core.config import Settings -from ..core.utils import first_non_empty logger = logging.getLogger("iaai_scraper.listing") VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) @@ -44,6 +43,7 @@ class ListingPageResult: class ListingCollector: + _DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE = 40 _NEXT_PAGE_SELECTORS: tuple[str, ...] = ( "a[aria-label*='Next']", "button[aria-label*='Next']", @@ -103,7 +103,33 @@ class ListingCollector: return None @staticmethod - def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool: + def _get_vehicle_link_fingerprint(page: Page, limit: int = 5) -> tuple[str, ...]: + try: + values = page.evaluate( + """ + (limit) => { + return Array.from(document.querySelectorAll("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']")) + .map((el) => (el.getAttribute('href') || '').trim()) + .filter(Boolean) + .slice(0, limit); + } + """, + limit, + ) + if not isinstance(values, list): + return tuple() + return tuple(str(value) for value in values if value) + except Exception: + return tuple() + + @staticmethod + def _wait_for_navigation_result( + page: Page, + old_first_href: str, + expected_page_number: int | None, + *, + old_fingerprint: tuple[str, ...] = (), + ) -> bool: if old_first_href: try: page.wait_for_function( @@ -117,9 +143,12 @@ class ListingCollector: except Exception: pass + new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page) if expected_page_number is not None: current_page = ListingCollector._get_current_page_number(page) - if current_page == expected_page_number: + if current_page == expected_page_number and ( + not old_fingerprint or (new_fingerprint and new_fingerprint != old_fingerprint) + ): return True try: @@ -127,11 +156,81 @@ class ListingCollector: except Exception: pass + if not new_fingerprint: + new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page) current_page = ListingCollector._get_current_page_number(page) + if new_fingerprint and old_fingerprint and new_fingerprint != old_fingerprint: + return current_page is None or expected_page_number is None or current_page == expected_page_number if expected_page_number is not None and current_page == expected_page_number: - return True + return not old_fingerprint - return not old_first_href + return not old_first_href and (not old_fingerprint or bool(new_fingerprint)) + + def _extract_next_page_href(self, page: Page, expected_page_number: int | None = None) -> str | None: + try: + href = page.evaluate( + """ + (expectedPageNumber) => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const disabled = (el) => { + if (!el) return true; + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + }; + + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')) + .filter((el) => visible(el) && !disabled(el)); + + const cleanHref = (el) => { + const href = (el?.getAttribute('href') || '').trim(); + if (!href || href.startsWith('javascript:') || href.startsWith('#')) { + return ''; + } + return href; + }; + + if (expectedPageNumber !== null && expectedPageNumber !== undefined) { + const numeric = controls.find((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === expectedPageNumber; + }); + const numericHref = cleanHref(numeric); + if (numericHref) { + return numericHref; + } + } + + const explicitNext = controls.find((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text); + }); + return cleanHref(explicitNext); + } + """, + expected_page_number, + ) + if not href: + return None + return urljoin(page.url or self.settings.home_url, str(href)) + except Exception: + return None + + @staticmethod + def _build_listing_page_url(url: str, page_number: int) -> str: + parsed = urlparse(url) + query_items = [ + (key, value) + for key, value in parse_qsl(parsed.query, keep_blank_values=True) + if key.lower() not in {"page", "pagenumber", "currentpage"} + ] + query_items.append(("page", str(page_number))) + return urlunparse(parsed._replace(query=urlencode(query_items))) def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None: url = url_override or self.settings.listing.cars_url @@ -202,6 +301,31 @@ class ListingCollector: # Короткая пауза вместо длинного sleep. time.sleep(1.0) + def _get_listing_html(self, page: Page, page_number: int) -> str: + try: + return page.locator("html").inner_html(timeout=5_000) + except Exception as exc: + logger.warning("listing html read failed on page %d: %s", page_number, exc) + return "" + + def _has_next_page_from_html(self, html: str, current_page_number: int | None = None) -> bool: + if not html: + return False + normalized = html.replace("\\/", "/") + if re.search( + r"rel\s*=\s*['\"]next['\"]|aria-label\s*=\s*['\"][^'\"]*next|title\s*=\s*['\"][^'\"]*next|class\s*=\s*['\"][^'\"]*next|icon-arrow-right|arrow-right|>\s*next\s*<|>\s*[›»>]\s*<", + normalized, + re.IGNORECASE, + ): + return True + if current_page_number is not None: + next_page = current_page_number + 1 + if re.search(rf">\s*{next_page}\s*<", normalized, re.IGNORECASE): + return True + if re.search(rf"page={next_page}(?:\D|$)", normalized, re.IGNORECASE): + return True + return False + def apply_filters( self, page: Page, @@ -279,74 +403,30 @@ class ListingCollector: return False def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: - # Считываем ссылки одним проходом по DOM. + # Для IAAI HTML/hydration-извлечение стабильнее, чем прямой DOM eval. self._accept_cookie_banner(page) self._wait_for_listing_content(page) - try: - raw_items = page.eval_on_selector_all( - "a[href], [data-href], [href]", - """ - (nodes) => nodes.map((node) => ({ - href: - node.getAttribute('href') || - node.getAttribute('data-href') || - node.getAttribute('data-url') || - '', - title: node.getAttribute('title') || node.getAttribute('aria-label') || '', - text: (node.textContent || '').trim(), - })) - """, - ) - except Exception as exc: - logger.warning("collect_current_page failed on page %d: %s", page_number, exc) - raw_items = [] - total = min(len(raw_items), self.settings.listing.page_link_limit) links: list[ListingVehicleLink] = [] seen: set[str] = set() - for idx in range(total): - item = raw_items[idx] if isinstance(raw_items[idx], dict) else {} - href = str(item.get("href") or "") - match = VEHICLE_HREF_RE.search(href) - if not match: - continue - lot_number = match.group(1) - absolute = urljoin(self.settings.home_url, match.group(0)) + html = self._get_listing_html(page, page_number) + for absolute, lot_number in self._extract_vehicle_links_from_html(html): if absolute in seen: continue seen.add(absolute) - title = first_non_empty([item.get("title"), item.get("text"), ""]) or "" - links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) + links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) if len(links) >= self.settings.listing.max_vehicles_per_run: break - # Fallback: на IAAI ссылки иногда не рендерятся как , - # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). - if not links: - try: - page.wait_for_timeout(1_500) - except Exception: - pass - try: - html = page.content() - except Exception as exc: - logger.debug("page.content() failed on page %d: %s", page_number, exc) - html = "" + if links: + logger.info( + "Page %d: recovered %d vehicle links from HTML fallback", + page_number, + len(links), + ) - for absolute, lot_number in self._extract_vehicle_links_from_html(html): - if absolute in seen: - continue - seen.add(absolute) - links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) - if len(links) >= self.settings.listing.max_vehicles_per_run: - break - - if links: - logger.info( - "Page %d: recovered %d vehicle links from HTML fallback", - page_number, - len(links), - ) - next_page_detected = self._has_next_page(page) + next_page_detected = self._has_next_page_from_html(html, current_page_number=page_number) + if not next_page_detected and not html: + next_page_detected = self._has_next_page(page) return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]: @@ -373,12 +453,59 @@ class ListingCollector: def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool: # Запоминаем первую ссылку текущей страницы для определения смены контента. old_first_href = "" + old_fingerprint: tuple[str, ...] = tuple() try: first_link = page.locator(VEHICLE_LINK_SELECTOR).first if first_link.count() > 0: old_first_href = first_link.get_attribute("href") or "" except Exception: pass + old_fingerprint = self._get_vehicle_link_fingerprint(page) + + if expected_page_number is not None and expected_page_number > 1: + direct_page_url = self._build_listing_page_url( + page.url or self.settings.listing.cars_url, + expected_page_number, + ) + try: + logger.debug("Navigating directly to listing page %d via URL: %s", expected_page_number, direct_page_url) + page.goto(direct_page_url, wait_until="domcontentloaded", timeout=15_000) + if self._wait_for_navigation_result( + page, + old_first_href, + expected_page_number, + old_fingerprint=old_fingerprint, + ): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("Direct page-number navigation failed for page %d via %s: %s", expected_page_number, direct_page_url, exc) + + next_href = self._extract_next_page_href(page, expected_page_number) + if next_href: + try: + logger.debug("Navigating directly to next listing page: %s", next_href) + page.goto(next_href, wait_until="domcontentloaded", timeout=15_000) + if self._wait_for_navigation_result( + page, + old_first_href, + expected_page_number, + old_fingerprint=old_fingerprint, + ): + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("Direct next-page navigation failed for %s: %s", next_href, exc) + + if ( + expected_page_number is not None + and expected_page_number >= self._DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE + ): + logger.warning( + "Deep pagination direct navigation failed for page %d; skipping flaky UI pagination fallbacks", + expected_page_number, + ) + return False for selector in self._NEXT_PAGE_SELECTORS: locator = page.locator(selector).first @@ -398,7 +525,12 @@ class ListingCollector: except Exception: continue - if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + if self._wait_for_navigation_result( + page, + old_first_href, + expected_page_number, + old_fingerprint=old_fingerprint, + ): self.pacer.after_page_change() return True @@ -458,7 +590,12 @@ class ListingCollector: """ )) if clicked: - if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + if self._wait_for_navigation_result( + page, + old_first_href, + expected_page_number, + old_fingerprint=old_fingerprint, + ): self.pacer.after_page_change() return True except Exception as exc: @@ -489,7 +626,12 @@ class ListingCollector: """ )) if clicked: - if self._wait_for_navigation_result(page, old_first_href, expected_page_number): + if self._wait_for_navigation_result( + page, + old_first_href, + expected_page_number, + old_fingerprint=old_fingerprint, + ): self.pacer.after_page_change() return True except Exception as exc: @@ -574,10 +716,18 @@ class ListingCollector: len(all_links) >= self.settings.listing.max_vehicles_per_run or self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination - or not page_result.next_page_detected ): break - if not self.go_to_next_page(page): + + blind_page_probe = not page_result.next_page_detected + if not self.go_to_next_page(page, expected_page_number=page_number + 1): + if blind_page_probe: + logger.info( + "Stopping pagination on page %d: direct page probe for %d failed and no next-page control was detected", + page_number, + page_number + 1, + ) + break break return { diff --git a/iaai_scraper/cli.py b/iaai_scraper/cli.py index c3b76d5..295c10f 100644 --- a/iaai_scraper/cli.py +++ b/iaai_scraper/cli.py @@ -38,6 +38,17 @@ def build_parser() -> argparse.ArgumentParser: sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json")) + # Новые команды для ingestion pipeline + subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap") + + fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates") + fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process") + + enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots") + enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process") + + subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)") + return parser @@ -64,6 +75,39 @@ def main() -> None: data = scraper.scrape_vehicle_detail(args.vehicle_url) elif args.command == "sync-vehicle": data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) + elif args.command == "sync-listing": + only_new = None if args.only_new is None else args.only_new == "true" + data = scraper.sync_listing( + make=args.make, + model=args.model, + lane=args.lane, + limit=args.limit, + only_new=only_new, + ) + elif args.command == "discover-vehicles": + from .discovery_service import DiscoveryService + discovery = DiscoveryService() + count = discovery.discover_new_vehicles() + print(f"Discovered {count} new vehicle candidates") + return + elif args.command == "fetch-pending": + from .fetch_service import FetchService + fetch = FetchService() + count = fetch.process_pending_candidates(limit=args.limit) + print(f"Successfully fetched {count} candidates") + return + elif args.command == "enrich-snapshots": + from .enrichment_service import EnrichmentService + enrichment = EnrichmentService() + count = enrichment.process_unparsed_snapshots(limit=args.limit) + print(f"Successfully enriched {count} snapshots") + return + elif args.command == "run-pipeline": + from .scheduler_service import SchedulerService + scheduler = SchedulerService() + scheduler.run_full_pipeline() + print("Pipeline completed") + return else: only_new = None if args.only_new is None else args.only_new == "true" data = scraper.sync_listing( diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py index e105ec1..f2a087e 100644 --- a/iaai_scraper/core/config.py +++ b/iaai_scraper/core/config.py @@ -106,6 +106,7 @@ class ListingConfig: cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999) max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000) + resume_max_nav_pages: int = _env_int("IAAI_LISTING_RESUME_MAX_NAV_PAGES", 90) page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500) include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True) collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False) @@ -118,6 +119,22 @@ class ListingConfig: listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "") +@dataclass(slots=True) +class DiscoveryConfig: + mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap") + hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh") + hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 3000) + sitemap_timeout_seconds: int = _env_int("IAAI_SITEMAP_TIMEOUT_SECONDS", 30) + sitemap_retry_attempts: int = _env_int("IAAI_SITEMAP_RETRY_ATTEMPTS", 4) + sitemap_retry_backoff_seconds: float = _env_float("IAAI_SITEMAP_RETRY_BACKOFF_SECONDS", 1.25) + sitemap_direct_probe_limit: int = _env_int("IAAI_SITEMAP_DIRECT_PROBE_LIMIT", 12) + sitemap_direct_probe_stop_after_misses: int = _env_int( + "IAAI_SITEMAP_DIRECT_PROBE_STOP_AFTER_MISSES", + 3, + ) + sitemap_use_curl_cffi: bool = _env_bool("IAAI_SITEMAP_USE_CURL_CFFI", True) + + # Список брендов IAAI для автоматической сегментации. # Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким. IAAI_DEFAULT_MAKES: tuple[str, ...] = ( @@ -217,6 +234,7 @@ class CeleryConfig: result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) + task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600) worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) @@ -224,6 +242,7 @@ class CeleryConfig: beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8) + parallel_segments: bool = _env_bool("IAAI_PARALLEL_SEGMENTS", False) block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True) @@ -278,6 +297,7 @@ class Settings: capture: CaptureConfig = field(default_factory=CaptureConfig) pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) listing: ListingConfig = field(default_factory=ListingConfig) + discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) database: DatabaseConfig = field(default_factory=DatabaseConfig) redis: RedisConfig = field(default_factory=RedisConfig) celery: CeleryConfig = field(default_factory=CeleryConfig) diff --git a/iaai_scraper/core/utils.py b/iaai_scraper/core/utils.py index 98b00a8..8c8cad5 100644 --- a/iaai_scraper/core/utils.py +++ b/iaai_scraper/core/utils.py @@ -17,7 +17,8 @@ def first_non_empty(values: Iterable[Any]) -> Any | None: return None -# Регулярные выражения для lot и price. +# Регулярные выражения для VIN, lot и price. +VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) LOT_RE = re.compile(r"\b(\d{7,10})\b") PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") diff --git a/iaai_scraper/discovery/__init__.py b/iaai_scraper/discovery/__init__.py new file mode 100644 index 0000000..c7d7e49 --- /dev/null +++ b/iaai_scraper/discovery/__init__.py @@ -0,0 +1,17 @@ +from .sitemap import ( + DEFAULT_SITEMAP_INDEX_URL, + SitemapDiscoveryError, + SitemapBlockedError, + SitemapDiscoveryResult, + discover_vehicle_urls_from_sitemap, + discover_vehicle_urls_from_sitemap_with_stats, +) + +__all__ = [ + "DEFAULT_SITEMAP_INDEX_URL", + "SitemapBlockedError", + "SitemapDiscoveryError", + "SitemapDiscoveryResult", + "discover_vehicle_urls_from_sitemap", + "discover_vehicle_urls_from_sitemap_with_stats", +] diff --git a/iaai_scraper/discovery/sitemap.py b/iaai_scraper/discovery/sitemap.py new file mode 100644 index 0000000..9471868 --- /dev/null +++ b/iaai_scraper/discovery/sitemap.py @@ -0,0 +1,441 @@ +from __future__ import annotations + +import gzip +import io +import logging +import random +import re +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from typing import Iterable +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin, urlsplit, urlunsplit +from urllib.request import Request, urlopen + +from ..core.config import Settings + +try: + from curl_cffi import requests as curl_requests +except ImportError: # pragma: no cover - optional runtime dependency guard + curl_requests = None + +logger = logging.getLogger("iaai_scraper.discovery.sitemap") + +DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" +_LOC_TAG_RE = re.compile(rb"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) +_XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL) +_BLOCK_MARKERS = ( + b"pardon our interruption", + b"incapsula", + b"please stand by", + b"_incapsula_resource", + b"as you were browsing something about your browser", +) +_HTML_MARKERS = (b" None: + self.settings = settings + self.discovery = settings.discovery + self.proxy_url = settings.proxy.server + self.proxy_auth = None + if settings.proxy.username: + password = settings.proxy.password or "" + self.proxy_auth = (settings.proxy.username, password) + self._transport_name = self._resolve_transport_name() + + @property + def transport_name(self) -> str: + return self._transport_name + + def _resolve_transport_name(self) -> str: + if self.discovery.sitemap_use_curl_cffi and curl_requests is not None: + return "curl_cffi" + return "urllib" + + def fetch(self, url: str) -> SitemapFetchResult: + attempts = max(1, int(self.discovery.sitemap_retry_attempts)) + last_exc: Exception | None = None + for attempt in range(1, attempts + 1): + try: + result = self._fetch_once(url) + if result.blocked: + raise SitemapBlockedError(f"Anti-bot page returned for {url}") + return result + except SitemapBlockedError as exc: + last_exc = exc + logger.warning( + "Sitemap fetch blocked (attempt %d/%d, transport=%s): %s", + attempt, + attempts, + self.transport_name, + url, + ) + except Exception as exc: + last_exc = exc + logger.warning( + "Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s", + attempt, + attempts, + self.transport_name, + url, + exc, + ) + if attempt >= attempts: + break + time.sleep(self._backoff_delay(attempt)) + if last_exc is None: + raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}") + if isinstance(last_exc, SitemapDiscoveryError): + raise last_exc + raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc + + def _fetch_once(self, url: str) -> SitemapFetchResult: + if self.transport_name == "curl_cffi": + return self._fetch_with_curl_cffi(url) + return self._fetch_with_urllib(url) + + def _build_headers(self) -> dict[str, str]: + headers = dict(_DEFAULT_HEADERS) + headers["User-Agent"] = self.settings.fingerprint.user_agent + return headers + + def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult: + assert curl_requests is not None + response = curl_requests.get( + url, + headers=self._build_headers(), + timeout=self.discovery.sitemap_timeout_seconds, + impersonate=random.choice(_CURL_IMPERSONATE_CHOICES), + proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None, + proxy_auth=self.proxy_auth, + allow_redirects=True, + ) + payload = self._decode_payload( + payload=response.content, + encoding=(response.headers.get("Content-Encoding") or ""), + url=url, + ) + blocked = _looks_like_block_page(payload) + return SitemapFetchResult( + url=url, + payload=payload, + source="curl_cffi", + blocked=blocked, + status_code=int(response.status_code), + content_type=response.headers.get("Content-Type"), + ) + + def _fetch_with_urllib(self, url: str) -> SitemapFetchResult: + request = Request(url, headers=self._build_headers()) + try: + with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response: + payload = response.read() + decoded = self._decode_payload( + payload=payload, + encoding=(response.headers.get("Content-Encoding") or ""), + url=url, + ) + return SitemapFetchResult( + url=url, + payload=decoded, + source="urllib", + blocked=_looks_like_block_page(decoded), + status_code=getattr(response, "status", None), + content_type=response.headers.get("Content-Type"), + ) + except HTTPError as exc: + payload = exc.read() if hasattr(exc, "read") else b"" + decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url) + blocked = exc.code in {403, 429} or _looks_like_block_page(decoded) + if blocked: + raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc + raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc + except URLError as exc: + raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc + + @staticmethod + def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes: + normalized = (encoding or "").lower().strip() + if normalized == "gzip" or url.lower().endswith(".gz"): + try: + return gzip.GzipFile(fileobj=io.BytesIO(payload)).read() + except OSError: + return payload + return payload + + def _backoff_delay(self, attempt: int) -> float: + base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds)) + jitter = random.uniform(0.05, 0.35) + return base * (2 ** (attempt - 1)) + jitter + + +def _is_vehicle_sitemap_url(url: str) -> bool: + lowered = url.strip().lower() + if "sitemapbranches" in lowered or "sitemapauctions" in lowered: + return False + return lowered.endswith(".xml") or lowered.endswith(".xml.gz") + + +def _normalize_vehicle_url(url: str) -> str: + parts = urlsplit(url.strip()) + return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) + + +def _local_name(tag: str) -> str: + if "}" in tag: + return tag.rsplit("}", 1)[1] + return tag + + +def _looks_like_block_page(payload: bytes) -> bool: + sample = payload[:8192].lower() + if any(marker in sample for marker in _BLOCK_MARKERS): + return True + if any(marker in sample for marker in _HTML_MARKERS) and b"" not in sample: + return True + return False + + +def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]: + for match in _LOC_TAG_RE.finditer(xml_bytes): + try: + value = match.group(1).decode("utf-8", errors="ignore").strip() + except Exception: + continue + if value: + yield value + + +def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]: + if _looks_like_block_page(xml_bytes): + raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML") + if not _XML_PREFIX_RE.search(xml_bytes[:256]): + fallback_values = list(_iter_loc_values_fallback(xml_bytes)) + if fallback_values: + yield from fallback_values + return + try: + root = ET.fromstring(xml_bytes) + except ET.ParseError as exc: + fallback_values = list(_iter_loc_values_fallback(xml_bytes)) + if fallback_values: + logger.warning( + "Falling back to regex sitemap loc extraction after XML parse error: %s", + exc, + ) + yield from fallback_values + return + raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc + + for element in root.iter(): + if _local_name(element.tag) != "loc": + continue + if not element.text: + continue + value = element.text.strip() + if value: + yield value + + +def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]: + result: list[str] = [] + seen: set[str] = set() + for url in urls: + normalized = _normalize_vehicle_url(url) + if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized: + continue + if normalized in seen: + continue + seen.add(normalized) + result.append(normalized) + return result + + +def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool: + return any("/vehicledetail/" in url.lower() for url in urls) + + +def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]: + base_dir = index_url.rsplit("/", 1)[0] + "/" + return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)] + + +def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]: + logger.info("Downloading sitemap index: %s", index_url) + try: + index_result = downloader.fetch(index_url) + sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)] + if sitemap_urls: + return sitemap_urls + except SitemapBlockedError as exc: + stats.blocked_sitemaps += 1 + logger.warning("Sitemap index blocked, switching to direct probe: %s", exc) + except SitemapDiscoveryError as exc: + stats.malformed_sitemaps += 1 + logger.warning("Sitemap index unusable, switching to direct probe: %s", exc) + + logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe") + return _probe_direct_sitemap_urls(index_url, downloader, stats) + + +def _probe_direct_sitemap_urls( + index_url: str, + downloader: _SitemapDownloader, + stats: SitemapDiscoveryStats, +) -> list[str]: + discovered: list[str] = [] + consecutive_misses = 0 + probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit) + + for sitemap_url in probe_urls: + try: + result = downloader.fetch(sitemap_url) + raw_urls = list(_iter_loc_values(result.payload)) + except SitemapBlockedError: + stats.blocked_sitemaps += 1 + consecutive_misses += 1 + stats.direct_probe_misses += 1 + if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses: + break + continue + except SitemapDiscoveryError: + consecutive_misses += 1 + stats.direct_probe_misses += 1 + if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses: + break + continue + + if not raw_urls: + consecutive_misses += 1 + stats.direct_probe_misses += 1 + if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses: + break + continue + + consecutive_misses = 0 + stats.direct_probe_hits += 1 + discovered.append(sitemap_url) + + return discovered + + +def _collect_vehicle_urls_from_sitemap( + sitemap_url: str, + downloader: _SitemapDownloader, + stats: SitemapDiscoveryStats, +) -> list[str]: + logger.info("Downloading sitemap: %s", sitemap_url) + try: + result = downloader.fetch(sitemap_url) + raw_urls = list(_iter_loc_values(result.payload)) + except SitemapBlockedError as exc: + stats.blocked_sitemaps += 1 + logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc) + return [] + except SitemapDiscoveryError as exc: + stats.malformed_sitemaps += 1 + logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) + return [] + + stats.fetched_sitemaps += 1 + vehicle_urls = _filter_vehicle_urls(raw_urls) + if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): + logger.info("Skipping non-vehicle sitemap %s", sitemap_url) + return [] + + logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) + return vehicle_urls + + +def discover_vehicle_urls_from_sitemap( + index_url: str = DEFAULT_SITEMAP_INDEX_URL, + *, + settings: Settings | None = None, +) -> list[str]: + result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings) + return result.vehicle_urls + + + +def discover_vehicle_urls_from_sitemap_with_stats( + index_url: str = DEFAULT_SITEMAP_INDEX_URL, + *, + settings: Settings | None = None, +) -> SitemapDiscoveryResult: + runtime_settings = settings or Settings() + downloader = _SitemapDownloader(runtime_settings) + stats = SitemapDiscoveryStats(transport=downloader.transport_name) + + sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats) + if not sitemap_urls: + raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs") + + all_vehicle_urls: list[str] = [] + for sitemap_url in sitemap_urls: + all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats)) + + deduped = _filter_vehicle_urls(all_vehicle_urls) + if not deduped: + raise SitemapDiscoveryError( + "No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue" + ) + + logger.info( + "Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)", + len(deduped), + stats.transport, + stats.fetched_sitemaps, + stats.blocked_sitemaps, + stats.malformed_sitemaps, + stats.direct_probe_hits, + stats.direct_probe_misses, + ) + return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats) diff --git a/iaai_scraper/discovery_service.py b/iaai_scraper/discovery_service.py new file mode 100644 index 0000000..bb91c4b --- /dev/null +++ b/iaai_scraper/discovery_service.py @@ -0,0 +1,129 @@ +import logging +from datetime import datetime, timezone +from typing import List + +from sqlalchemy.orm import Session + +from .core.config import settings +from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError +from .storage.db import get_db_session +from .storage.models import VehicleCandidate + +logger = logging.getLogger("iaai_scraper.discovery_service") + + +class DiscoveryService: + """Сервис для обнаружения новых автомобилей через sitemap IAAI.""" + + def __init__(self): + self.db: Session = get_db_session() + + def discover_new_vehicles(self, max_urls: int = None) -> int: + """ + Обнаруживает новые URL автомобилей и добавляет их в vehicle_candidates. + + Args: + max_urls: Максимальное количество URL для обработки (None = все) + + Returns: + Количество добавленных кандидатов + """ + try: + logger.info("Starting vehicle discovery from sitemap") + result = discover_vehicle_urls_from_sitemap_with_stats() + + if not result.urls: + logger.info("No new URLs discovered") + return 0 + + # Ограничиваем количество URL + urls_to_process = result.urls[:max_urls] if max_urls else result.urls + logger.info(f"Discovered {len(urls_to_process)} potential vehicle URLs (from {len(result.urls)} total)") + + # Фильтруем уже существующие кандидаты + existing_urls = self._get_existing_candidate_urls(urls_to_process) + new_urls = [url for url in urls_to_process if url not in existing_urls] + + if not new_urls: + logger.info("All discovered URLs already exist as candidates") + return 0 + + # Добавляем новых кандидатов + added_count = self._add_candidates(new_urls) + logger.info(f"Added {added_count} new vehicle candidates") + + return added_count + + except SitemapDiscoveryError as e: + logger.error(f"Sitemap discovery failed: {e}") + raise + except Exception as e: + logger.error(f"Unexpected error during discovery: {e}") + raise + + def _get_existing_candidate_urls(self, urls: List[str]) -> set: + """Получает множество уже существующих URL кандидатов.""" + if not urls: + return set() + + # Разбиваем на батчи для эффективности + batch_size = 1000 + existing = set() + + for i in range(0, len(urls), batch_size): + batch = urls[i:i + batch_size] + result = self.db.query(VehicleCandidate.url).filter( + VehicleCandidate.url.in_(batch) + ).all() + existing.update(row[0] for row in result) + + return existing + + def _add_candidates(self, urls: List[str]) -> int: + """Добавляет новые кандидаты в базу данных.""" + now = datetime.now(timezone.utc) + candidates = [ + VehicleCandidate( + url=url, + discovered_at=now, + status="pending", + priority=0 + ) + for url in urls + ] + + self.db.add_all(candidates) + self.db.commit() + + return len(candidates) + + def get_pending_candidates(self, limit: int = 100) -> List[VehicleCandidate]: + """Получает кандидатов в статусе 'pending' для обработки.""" + return self.db.query(VehicleCandidate).filter( + VehicleCandidate.status == "pending" + ).order_by(VehicleCandidate.priority.desc(), VehicleCandidate.discovered_at).limit(limit).all() + + def mark_candidate_processing(self, candidate_id: int): + """Помечает кандидата как обрабатываемого.""" + self.db.query(VehicleCandidate).filter( + VehicleCandidate.id == candidate_id + ).update({ + "status": "processing", + "last_attempt_at": datetime.now(timezone.utc), + "attempts": VehicleCandidate.attempts + 1 + }) + self.db.commit() + + def mark_candidate_processed(self, candidate_id: int): + """Помечает кандидата как обработанного.""" + self.db.query(VehicleCandidate).filter( + VehicleCandidate.id == candidate_id + ).update({"status": "processed"}) + self.db.commit() + + def mark_candidate_failed(self, candidate_id: int): + """Помечает кандидата как неудачного.""" + self.db.query(VehicleCandidate).filter( + VehicleCandidate.id == candidate_id + ).update({"status": "failed"}) + self.db.commit() \ No newline at end of file diff --git a/iaai_scraper/enrichment_service.py b/iaai_scraper/enrichment_service.py new file mode 100644 index 0000000..fa36391 --- /dev/null +++ b/iaai_scraper/enrichment_service.py @@ -0,0 +1,140 @@ +import json +import logging +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy.orm import Session + +from .core.config import settings +from .parsing.mapper import CarMapper +from .parsing.parser import VehicleParser +from .storage.db import get_db_session +from .storage.models import VehicleRawSnapshot, VehicleParseResult, Car +from .storage.schemas import CarRecord + +logger = logging.getLogger("iaai_scraper.enrichment_service") + + +class EnrichmentService: + """Сервис для парсинга сырых данных и обогащения автомобилей.""" + + def __init__(self): + self.db: Session = get_db_session() + self.parser = VehicleParser() + self.mapper = CarMapper() + + def enrich_vehicle(self, snapshot: VehicleRawSnapshot) -> Optional[VehicleParseResult]: + """ + Парсит snapshot и сохраняет результат. + + Args: + snapshot: Raw snapshot для парсинга + + Returns: + VehicleParseResult если парсинг успешен + """ + logger.info(f"Enriching snapshot {snapshot.id} for candidate {snapshot.candidate_id}") + + if not snapshot.success or not snapshot.raw_data: + logger.warning(f"Snapshot {snapshot.id} has no data to parse") + return self._save_parse_result(snapshot.id, False, None, "No raw data available") + + try: + # Парсим данные + parsed_data = self._parse_raw_data(snapshot.raw_data) + if not parsed_data: + return self._save_parse_result(snapshot.id, False, None, "Parsing failed") + + # Маппим в CarRecord + car_record = self._map_to_car_record(parsed_data) + if not car_record: + return self._save_parse_result(snapshot.id, False, None, "Mapping failed") + + # Сохраняем в cars таблицу + self._save_car(car_record) + + # Сохраняем успешный результат парсинга + return self._save_parse_result(snapshot.id, True, json.dumps(parsed_data)) + + except Exception as e: + error_msg = f"Unexpected error during enrichment: {str(e)}" + logger.error(f"Error enriching snapshot {snapshot.id}: {error_msg}") + return self._save_parse_result(snapshot.id, False, None, error_msg) + + def _parse_raw_data(self, raw_data: str) -> Optional[dict]: + """Парсит сырые данные в словарь.""" + try: + # Если это JSON от browser capture + if raw_data.strip().startswith('{'): + data = json.loads(raw_data) + # Извлекаем vehicle_summary из scrape result + return data.get("vehicle_summary", {}) + + # Если HTML, используем VehicleParser + parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url") + return parsed.get("vehicle_summary", {}) + + except json.JSONDecodeError: + # Если не JSON, пробуем как HTML + try: + parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url") + return parsed.get("vehicle_summary", {}) + except Exception: + return None + + def _map_to_car_record(self, parsed_data: dict) -> Optional[CarRecord]: + """Маппит parsed data в CarRecord.""" + try: + # Используем CarMapper + payload_insights = {} # TODO: extract from raw data if needed + return self.mapper.map_to_car_record("dummy_url", parsed_data, payload_insights) + except Exception as e: + logger.error(f"Mapping failed: {e}") + return None + + def _save_car(self, car_record: CarRecord): + """Сохраняет CarRecord в базу данных.""" + # Используем PersistenceService + from .storage.db import PersistenceService + from .core.config import settings + persistence = PersistenceService(settings) + persistence.create_tables() + upsert_result = persistence.upsert_car(car_record) + logger.info(f"Car upserted: {upsert_result}") + + def _save_parse_result(self, snapshot_id: int, success: bool, + parsed_data: Optional[str], error_message: Optional[str] = None) -> VehicleParseResult: + """Сохраняет результат парсинга.""" + result = VehicleParseResult( + snapshot_id=snapshot_id, + success=success, + parsed_data=parsed_data, + error_message=error_message + ) + self.db.add(result) + self.db.commit() + self.db.refresh(result) + return result + + def process_unparsed_snapshots(self, limit: int = 10) -> int: + """ + Обрабатывает snapshots без результатов парсинга. + + Returns: + Количество успешно обогащенных snapshots + """ + # Находим snapshots без parse results + snapshots = self.db.query(VehicleRawSnapshot).outerjoin( + VehicleParseResult, VehicleRawSnapshot.id == VehicleParseResult.snapshot_id + ).filter( + VehicleRawSnapshot.success == True, + VehicleParseResult.id.is_(None) + ).limit(limit).all() + + enriched_count = 0 + for snapshot in snapshots: + result = self.enrich_vehicle(snapshot) + if result and result.success: + enriched_count += 1 + + return enriched_count \ No newline at end of file diff --git a/iaai_scraper/fetch_service.py b/iaai_scraper/fetch_service.py new file mode 100644 index 0000000..39874ea --- /dev/null +++ b/iaai_scraper/fetch_service.py @@ -0,0 +1,119 @@ +import logging +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy.orm import Session + +from .core.config import settings +from .scraper import IAAIScraper +from .storage.db import get_db_session +from .storage.models import VehicleCandidate, VehicleRawSnapshot + +logger = logging.getLogger("iaai_scraper.fetch_service") + + +class FetchService: + """Сервис для захвата сырых данных автомобилей (HTTP/browser fallback).""" + + def __init__(self): + self.db: Session = get_db_session() + self.scraper = IAAIScraper() + + def fetch_vehicle_data(self, candidate: VehicleCandidate) -> Optional[VehicleRawSnapshot]: + """ + Захватывает данные для кандидата автомобиля. + + Args: + candidate: Кандидат для обработки + + Returns: + VehicleRawSnapshot если захват успешен, None если неудача + """ + logger.info(f"Fetching data for candidate {candidate.id}: {candidate.url}") + + try: + # Сначала пытаемся HTTP fast-path + raw_data = self._try_http_capture(candidate.url) + if raw_data: + return self._save_snapshot(candidate.id, "http", True, raw_data) + + # Если HTTP не сработал, используем browser fallback + logger.info(f"HTTP failed for {candidate.url}, trying browser fallback") + raw_data = self._try_browser_capture(candidate.url) + if raw_data: + return self._save_snapshot(candidate.id, "browser", True, raw_data) + + # Оба метода failed + logger.warning(f"Both HTTP and browser capture failed for {candidate.url}") + self._save_snapshot(candidate.id, "browser", False, None, "Both capture methods failed") + return None + + except Exception as e: + error_msg = f"Unexpected error during capture: {str(e)}" + logger.error(f"Error fetching {candidate.url}: {error_msg}") + self._save_snapshot(candidate.id, "unknown", False, None, error_msg) + return None + + def _try_http_capture(self, url: str) -> Optional[str]: + """Пытается захватить данные через HTTP.""" + try: + # Используем существующий метод из scraper + # Но нам нужно инициализировать scraper с сессией + # Пока заглушка - интегрируем позже + # Для теста вернем None, чтобы использовать browser + return None + except Exception as e: + logger.debug(f"HTTP capture failed for {url}: {e}") + return None + + def _try_browser_capture(self, url: str) -> Optional[str]: + """Пытается захватить данные через browser.""" + try: + # Используем scrape_vehicle_detail из scraper + with IAAIScraper() as scraper: + result = scraper.scrape_vehicle_detail(url) + # Возвращаем HTML или JSON данные + return result.get("raw_html") or json.dumps(result) + except Exception as e: + logger.debug(f"Browser capture failed for {url}: {e}") + return None + + def _save_snapshot(self, candidate_id: int, method: str, success: bool, + raw_data: Optional[str], error_message: Optional[str] = None) -> VehicleRawSnapshot: + """Сохраняет snapshot в базу данных.""" + snapshot = VehicleRawSnapshot( + candidate_id=candidate_id, + method=method, + success=success, + raw_data=raw_data, + error_message=error_message + ) + self.db.add(snapshot) + self.db.commit() + self.db.refresh(snapshot) + return snapshot + + def process_pending_candidates(self, limit: int = 10) -> int: + """ + Обрабатывает ожидающих кандидатов. + + Returns: + Количество успешно обработанных кандидатов + """ + from .discovery_service import DiscoveryService + discovery = DiscoveryService() + + candidates = discovery.get_pending_candidates(limit) + processed_count = 0 + + for candidate in candidates: + discovery.mark_candidate_processing(candidate.id) + + snapshot = self.fetch_vehicle_data(candidate) + if snapshot and snapshot.success: + discovery.mark_candidate_processed(candidate.id) + processed_count += 1 + else: + discovery.mark_candidate_failed(candidate.id) + + return processed_count \ No newline at end of file diff --git a/iaai_scraper/scheduler_service.py b/iaai_scraper/scheduler_service.py new file mode 100644 index 0000000..07159af --- /dev/null +++ b/iaai_scraper/scheduler_service.py @@ -0,0 +1,69 @@ +import logging +import time +from datetime import datetime, timezone, timedelta + +from .core.config import settings +from .discovery_service import DiscoveryService +from .fetch_service import FetchService +from .enrichment_service import EnrichmentService + +logger = logging.getLogger("iaai_scraper.scheduler_service") + + +class SchedulerService: + """Сервис для планирования и координации ingestion pipeline.""" + + def __init__(self): + self.discovery = DiscoveryService() + self.fetch = FetchService() + self.enrichment = EnrichmentService() + + def run_full_pipeline(self): + """Запускает полный цикл ingestion: discovery -> fetch -> enrichment.""" + logger.info("Starting full ingestion pipeline") + + try: + # 1. Discovery phase + logger.info("Phase 1: Discovery") + discovered_count = self.discovery.discover_new_vehicles() + logger.info(f"Discovered {discovered_count} new candidates") + + # 2. Fetch phase + logger.info("Phase 2: Fetch") + fetched_count = self.fetch.process_pending_candidates(limit=50) + logger.info(f"Successfully fetched {fetched_count} candidates") + + # 3. Enrichment phase + logger.info("Phase 3: Enrichment") + enriched_count = self.enrichment.process_unparsed_snapshots(limit=50) + logger.info(f"Successfully enriched {enriched_count} snapshots") + + logger.info("Ingestion pipeline completed") + + except Exception as e: + logger.error(f"Pipeline failed: {e}") + raise + + def run_continuous_pipeline(self, interval_minutes: int = 30): + """Запускает непрерывный цикл ingestion с интервалом.""" + logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals") + + while True: + try: + self.run_full_pipeline() + except Exception as e: + logger.error(f"Pipeline iteration failed: {e}") + + logger.info(f"Sleeping for {interval_minutes} minutes") + time.sleep(interval_minutes * 60) + + def run_targeted_enrichment(self): + """Запускает только enrichment для существующих snapshots.""" + logger.info("Running targeted enrichment") + enriched_count = self.enrichment.process_unparsed_snapshots(limit=100) + logger.info(f"Enriched {enriched_count} snapshots") + + def cleanup_old_data(self, days_to_keep: int = 30): + """Очищает старые данные (опционально).""" + # TODO: implement if needed + pass \ No newline at end of file diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py index 94bdd98..ba9caa5 100644 --- a/iaai_scraper/scraper.py +++ b/iaai_scraper/scraper.py @@ -1,13 +1,12 @@ import json import logging import os -import random import re import signal import time import uuid import html as html_module -from concurrent.futures import Future, ThreadPoolExecutor, as_completed +from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable @@ -16,18 +15,12 @@ from urllib.request import Request, urlopen import urllib3 -try: - import orjson as _orjson # ~3-5× быстрее stdlib json на парсинге - _HAS_ORJSON = True -except ImportError: - _orjson = None - _HAS_ORJSON = False - from playwright.sync_api import Error as PlaywrightError from playwright.sync_api import BrowserContext, Page, sync_playwright from playwright.sync_api import TimeoutError as PlaywrightTimeoutError from .browser import BrowserFactory, HumanPacer, NetworkCapture +from .discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats from .core.config import Settings, settings, parse_listing_segments from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError from .core.logs import set_trace_id, setup_logging @@ -45,69 +38,6 @@ VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) HTML_TAG_RE = re.compile(r"<[^>]+>") SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) -# ── Anti-stall watchdog ── -# Жёсткие верхние границы на per-page операции браузера. Если Playwright "тихо" завис -# (частая реакция IAAI на затяжной скрапинг — page hangs без raise), SIGALRM вернёт -# управление и существующая recover-логика (_reopen_listing_and_resume) стартует -# новый context с нуля. -_PAGE_COLLECT_TIMEOUT_S = 90 -_PAGE_NEXT_TIMEOUT_S = 60 -# Проактивная ротация контекста отключена: reactive-восстановление -# (retry при пустой странице + _reopen_listing_and_resume) уже закрывает -# anti-bot соскоки, а проактивная ротация требовала долистывать кликами -# после reopen и конфликтовала с лимитом max_nav_pages в recovery-пути. -# Включить можно переменной окружения IAAI_CONTEXT_ROTATE_EVERY_PAGES. -_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("IAAI_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) - - -class PageOperationTimeoutError(Exception): - """Browser page operation exceeded per-op watchdog timeout.""" - - -class _PageOpWatchdog: - """SIGALRM-based watchdog. - - Работает только в главном потоке процесса (Celery prefork child — это ок). - В других контекстах — no-op. - """ - - def __init__(self, seconds: int, label: str) -> None: - self.seconds = max(1, int(seconds)) - self.label = label - self._old_handler = None - self._active = False - - def _handler(self, signum, frame): # noqa: ARG002 - raise PageOperationTimeoutError( - f"Page operation '{self.label}' exceeded {self.seconds}s watchdog" - ) - - def __enter__(self): - import threading as _threading - if _threading.current_thread() is not _threading.main_thread(): - return self - if not hasattr(signal, "SIGALRM"): - return self - try: - self._old_handler = signal.signal(signal.SIGALRM, self._handler) - signal.alarm(self.seconds) - self._active = True - except (ValueError, OSError): - # signal можно выставлять только из main thread; в остальном пропускаем. - self._active = False - return self - - def __exit__(self, exc_type, exc, tb): - if not self._active: - return False - try: - signal.alarm(0) - if self._old_handler is not None: - signal.signal(signal.SIGALRM, self._old_handler) - except (ValueError, OSError): - pass - return False - class IAAIScraper: @@ -191,24 +121,14 @@ class IAAIScraper: self.car_mapper = CarMapper() self.persistence = PersistenceService(self.settings) self._shutdown_requested = False - # HTTP pool: при parallel_tabs=24 и concurrency=4 пик ≈ 96 конкурентных сокетов; - # даём запас до 256, чтобы не упираться в PoolError под всплесками PX/retry. - # TCP_NODELAY выключает Nagle: для коротких HTTPS-запросов с keep-alive - # это снижает tail-latency на десятки ms. - import socket as _socket - _socket_options = [ - (_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1), - (_socket.SOL_SOCKET, _socket.SO_KEEPALIVE, 1), - ] + self._progress_callback: Callable[[str, dict[str, Any]], None] | None = None proxy_url = self.settings.proxy.server if proxy_url: _proxy_kwargs: dict = { - "num_pools": 8, - "maxsize": 256, - "block": False, + "num_pools": 4, + "maxsize": 64, "retries": False, - "timeout": urllib3.Timeout(connect=5, read=20), - "socket_options": _socket_options, + "timeout": urllib3.Timeout(connect=5, read=10), } if self.settings.proxy.username: _proxy_kwargs["proxy_headers"] = urllib3.make_headers( @@ -218,12 +138,10 @@ class IAAIScraper: logger.info("HTTP fast-path using proxy: %s", proxy_url) else: self._http_pool = urllib3.PoolManager( - num_pools=8, - maxsize=256, - block=False, + num_pools=4, + maxsize=64, retries=False, - timeout=urllib3.Timeout(connect=5, read=20), - socket_options=_socket_options, + timeout=urllib3.Timeout(connect=5, read=10), ) def _new_trace_id(self, prefix: str) -> str: @@ -232,6 +150,18 @@ class IAAIScraper: set_trace_id(trace_id) return trace_id + def set_progress_callback(self, callback: Callable[[str, dict[str, Any]], None] | None) -> None: + self._progress_callback = callback + + def _emit_progress(self, stage: str, **payload: Any) -> None: + callback = self._progress_callback + if callback is None: + return + try: + callback(stage, payload) + except Exception: + logger.debug("progress callback failed for stage=%s", stage, exc_info=True) + def __enter__(self) -> "IAAIScraper": if self.playwright is None: self.playwright = sync_playwright().start() @@ -321,6 +251,133 @@ class IAAIScraper: vehicle_urls.append(normalized) return vehicle_urls + def _should_use_sitemap_discovery( + self, + *, + make: str | None, + model: str | None, + listing_url: str | None, + year_min: int | None, + year_max: int | None, + limit: int | None, + effective_only_new: bool, + ) -> bool: + # Полный unfiltered scan всегда должен идти через sitemap. + # Listing-пагинация на IAAI медленная, нестабильная и может зависать + # на resume/recovery даже когда raw HTTP по карточкам работает быстро. + return ( + make is None + and model is None + and listing_url is None + and year_min is None + and year_max is None + and (limit is None or limit <= 0) + and not effective_only_new + ) + + def _sync_listing_via_sitemap( + self, + *, + lane: str, + limit: int | None, + effective_only_new: bool, + ) -> dict[str, Any]: + del effective_only_new # sitemap full-scan path is used only for full discovery. + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=self.settings) + vehicle_urls = self._dedupe_urls(discovery_result.vehicle_urls) + if limit is not None and limit > 0: + vehicle_urls = vehicle_urls[:limit] + + batch_size = self.settings.celery.batch_size + total = len(vehicle_urls) + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + logger.info("Sitemap-based full discovery found %d vehicle URLs", total) + self._emit_progress( + "sitemap_discovery_done", + urls_found=total, + batch_size=batch_size, + transport=discovery_result.stats.transport, + fetched_sitemaps=discovery_result.stats.fetched_sitemaps, + blocked_sitemaps=discovery_result.stats.blocked_sitemaps, + malformed_sitemaps=discovery_result.stats.malformed_sitemaps, + direct_probe_hits=discovery_result.stats.direct_probe_hits, + direct_probe_misses=discovery_result.stats.direct_probe_misses, + ) + + for batch_start in range(0, total, batch_size): + batch_urls = vehicle_urls[batch_start:batch_start + batch_size] + self._emit_progress( + "listing_batch_start", + source="sitemap", + batch_offset=batch_start, + batch_size=len(batch_urls), + pending_urls=max(0, total - batch_start), + batch_first_url=batch_urls[0] if batch_urls else None, + ) + try: + batch_result = self.sync_batch(batch_urls, lane=lane) + cars_upserted += batch_result.get("cars_upserted", 0) + cars_failed += batch_result.get("cars_failed", 0) + images_upserted += batch_result.get("images_upserted", 0) + failures.extend(batch_result.get("failures", [])) + except Exception as batch_exc: + logger.error( + "Sitemap batch %d-%d failed: %s", + batch_start + 1, + batch_start + len(batch_urls), + batch_exc, + ) + cars_failed += len(batch_urls) + failures.append({ + "vehicle_url": f"sitemap_batch_{batch_start}", + "error": str(batch_exc), + }) + self._emit_progress( + "listing_batch_done", + source="sitemap", + batch_offset=batch_start, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + pending_urls=max(0, total - (batch_start + len(batch_urls))), + ) + + return { + "listing": { + "status": "ok", + "listing_url": self.settings.listing.cars_url, + "applied_filters": { + "make": None, + "model": None, + "year_min": None, + "year_max": None, + }, + "pages_collected": 0, + "vehicles_collected": total, + "vehicle_urls": vehicle_urls, + "early_stopped": False, + "truncated_by_time_budget": False, + "pagination_interrupted": False, + "pages": [], + "source": "sitemap", + "transport": discovery_result.stats.transport, + "blocked_sitemaps": discovery_result.stats.blocked_sitemaps, + "malformed_sitemaps": discovery_result.stats.malformed_sitemaps, + "direct_probe_hits": discovery_result.stats.direct_probe_hits, + "direct_probe_misses": discovery_result.stats.direct_probe_misses, + }, + "total": total, + "skipped_existing": 0, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "failures": failures, + "all_listing_origin_urls": set(vehicle_urls), + } + def _filter_known_urls(self, vehicle_urls: list[str]) -> tuple[list[str], int]: url_to_origin_id = { url: self._extract_db_origin_id_from_url(url) @@ -427,8 +484,10 @@ class IAAIScraper: listing_url: str | None = None, year_min: int | None = None, year_max: int | None = None, - max_nav_pages: int = 10, + max_nav_pages: int | None = None, ) -> tuple[Page, dict[str, str | int | None]]: + if max_nav_pages is None: + max_nav_pages = max(1, self.settings.listing.resume_max_nav_pages) # Ограничиваем глубину навигации: если до цели > max_nav_pages кликов — не пытаемся. if target_page_number > max_nav_pages + 1: raise ListingResumeError( @@ -523,6 +582,7 @@ class IAAIScraper: failures: list[dict[str, str]] = [] early_stopped = False truncated_by_time_budget = False + pagination_interrupted = False known_origin_ids: set[str] | None = None threshold = self.settings.listing.early_stop_threshold @@ -599,44 +659,32 @@ class IAAIScraper: "all_listing_origin_urls": all_listing_origin_urls, } - # ── Pipeline-режим: батчи (HTTP + DB upsert) выполняются в фоне, - # пока главный поток продолжает собирать listing-страницы через - # Playwright. Это убирает простой Playwright во время HTTP-фазы - # (~50-65 сек на 400 машин) и даёт ~25-30% к throughput. - # max_workers=1: батчи исполняются последовательно, но ВНЕ - # главного потока, чтобы не блокировать сбор listing-а. - # Browser fallback откладывается (skip_browser_fallback=True), - # потому что Playwright не thread-safe — обработаем после loop'а. - batch_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="batch-pipeline") - pending_future: Future | None = None - pending_future_meta: tuple[int, int] | None = None # (batch_start, batch_size) - deferred_fallbacks: list[tuple[str, int]] = [] - - def _drain_pending_future() -> bool: - """Waits for in-flight batch and accumulates its result. - - Returns False if a non-recoverable error occurred and remaining - batches should be aborted (mirrors the legacy contract). - """ - nonlocal pending_future, pending_future_meta + def _process_pending_batch(batch_urls: list[str], batch_start: int) -> bool: nonlocal cars_upserted, cars_failed, images_upserted, failures - if pending_future is None: + if not batch_urls: return True - future = pending_future - meta = pending_future_meta or (0, 0) - pending_future = None - pending_future_meta = None - batch_start, batch_size_actual = meta + + logger.info( + "Processing batch %d-%d of %d", + batch_start + 1, + batch_start + len(batch_urls), + total, + ) try: - batch_result = future.result() + batch_result = self.sync_batch(batch_urls, lane=lane) + cars_upserted += batch_result.get("cars_upserted", 0) + cars_failed += batch_result.get("cars_failed", 0) + images_upserted += batch_result.get("images_upserted", 0) + failures.extend(batch_result.get("failures", [])) + return True except PlaywrightError as pw_exc: logger.warning( "Batch %d-%d browser crashed: %s. Reinitializing browser context.", batch_start + 1, - batch_start + batch_size_actual, + batch_start + len(batch_urls), pw_exc, ) - cars_failed += batch_size_actual + cars_failed += len(batch_urls) failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(pw_exc)}) try: self._new_context() @@ -649,51 +697,12 @@ class IAAIScraper: logger.error( "Batch %d-%d failed: %s. Continuing with next batch.", batch_start + 1, - batch_start + batch_size_actual, + batch_start + len(batch_urls), batch_exc, ) - cars_failed += batch_size_actual + cars_failed += len(batch_urls) failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)}) return True - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - deferred_fallbacks.extend(batch_result.get("deferred_fallback_urls", [])) - return True - - def _process_pending_batch(batch_urls: list[str], batch_start: int) -> bool: - nonlocal pending_future, pending_future_meta - # Drain previous batch first (if still running). - if not _drain_pending_future(): - return False - if not batch_urls: - return True - logger.info( - "Processing batch %d-%d of %d (pipelined)", - batch_start + 1, - batch_start + len(batch_urls), - total, - ) - # Cookie header обязательно вычисляем в главном потоке — Playwright - # context.cookies() использует greenlet, который нельзя дёргать из background thread'а. - cookie_header = self._cookie_header_for_context() - pending_future = batch_executor.submit( - self.sync_batch, - batch_urls, - lane=lane, - skip_browser_fallback=True, - cookie_header_override=cookie_headerkground thread'а. - cookie_header = self._cookie_header_for_context() - pending_future = batch_executor.submit( - self.sync_batch, - batch_urls, - lane=lane, - skip_browser_fallback=True, - cookie_header_override=cookie_header, - ) - pending_future_meta = (batch_start, len(batch_urls)) - return True page = None try: @@ -704,78 +713,61 @@ class IAAIScraper: year_min=year_min, year_max=year_max, ) + self._emit_progress( + "listing_opened", + make=make, + model=model, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) - pages_since_rotation = 0 for page_number in range(1, self.settings.listing.max_pages_per_run + 1): - # Проактивная ротация контекста (опционально, по умолчанию выключена). - # Если включена — требует долистывания до page_number после reopen, - # поэтому max_nav_pages поднят под реальную глубину. - if _CONTEXT_ROTATE_EVERY_PAGES > 0 and pages_since_rotation >= _CONTEXT_ROTATE_EVERY_PAGES: - logger.warning( - "Rotating browser context at page %d (every %d pages) to reset anti-bot state", - page_number, _CONTEXT_ROTATE_EVERY_PAGES, - ) + self._emit_progress( + "listing_collect_start", + page_number=page_number, + pending_urls=len(pending_urls), + total_new=total, + ) + + # Защита от crashes при сборе данных страницы + page_result = None + collect_attempts = 0 + max_collect_attempts = 3 + + while collect_attempts < max_collect_attempts and page_result is None: + collect_attempts += 1 try: - page.close() - except Exception: - pass - page = None - try: - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=max(page_number, 500), - ) - pages_since_rotation = 0 - except ListingResumeError as resume_exc: - logger.warning( - "Context rotation could not resume at page %d (%s) — stopping segment", - page_number, resume_exc, - ) - break - - try: - with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number}"): page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - except (PageOperationTimeoutError, PlaywrightError) as op_exc: - logger.warning( - "Page %d collect stalled/failed (%s) — reopening listing and resuming", - page_number, op_exc, - ) - try: - page.close() - except Exception: - pass - page = None - try: - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=max(page_number, 500), - ) - pages_since_rotation = 0 - with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number} (after reopen)"): - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - except (ListingResumeError, PageOperationTimeoutError, PlaywrightError) as resume_exc: - logger.warning( - "Cannot recover at page %d (%s) — stopping pagination for this segment", - page_number, resume_exc, - ) - break - + except (PlaywrightError, PlaywrightTimeoutError, Exception) as collect_exc: + logger.warning("Exception during page collection (page %d, attempt %d/%d): %s", + page_number, collect_attempts, max_collect_attempts, collect_exc) + if collect_attempts < max_collect_attempts: + logger.info("Retrying page collection after 5 seconds...") + time.sleep(5) + try: + page.reload(wait_until="domcontentloaded", timeout=30_000) + except Exception as reload_exc: + logger.debug("Page reload failed during collection retry: %s", reload_exc) + else: + logger.error("All collection attempts failed for page %d — stopping pagination", page_number) + pagination_interrupted = True + page = None + break + + if page_result is None: + break + + self._emit_progress( + "listing_collect_done", + page_number=page_number, + links_found=len(page_result.vehicle_links), + next_page_detected=page_result.next_page_detected, + ) pages_info.append({ "page_number": page_result.page_number, "links_found": len(page_result.vehicle_links), }) - pages_since_rotation += 1 page_urls = self._extract_page_urls( page_result, @@ -785,6 +777,7 @@ class IAAIScraper: ) if not page_urls: + self._emit_progress("listing_empty_page_recovery", page_number=page_number) page_result, page_urls = self._recover_empty_listing_page( page, page_number=page_number, @@ -795,6 +788,7 @@ class IAAIScraper: ) if not page_urls and page_number > 1: logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number) + self._emit_progress("listing_resume_start", page_number=page_number) page.close() try: page, _ = self._reopen_listing_and_resume( @@ -804,7 +798,6 @@ class IAAIScraper: listing_url=listing_url, year_min=year_min, year_max=year_max, - max_nav_pages=max(page_number, 500), ) page_result = self.listing_collector.collect_current_page(page, page_number=page_number) page_urls = self._extract_page_urls( @@ -818,6 +811,7 @@ class IAAIScraper: "Cannot resume at page %d (%s) — stopping pagination for this segment", page_number, resume_exc, ) + pagination_interrupted = True page = None page_urls = [] if not page_urls: @@ -882,34 +876,52 @@ class IAAIScraper: while len(pending_urls) >= batch_size: batch_urls = pending_urls[:batch_size] + self._emit_progress( + "listing_batch_start", + page_number=page_number, + pending_urls=len(pending_urls), + batch_size=len(batch_urls), + batch_first_url=batch_urls[0] if batch_urls else None, + ) if not _process_pending_batch(batch_urls, cars_upserted + cars_failed): pending_urls = [] break pending_urls = pending_urls[batch_size:] + self._emit_progress( + "listing_batch_done", + page_number=page_number, + pending_urls=len(pending_urls), + cars_upserted=cars_upserted, + cars_failed=cars_failed, + ) if limit is not None and limit > 0 and total >= limit: break if ( self.settings.listing.collect_current_page_only or not self.settings.listing.include_pagination - or not page_result.next_page_detected ): break + self._emit_progress("listing_next_page_start", from_page=page_number, to_page=page_number + 1) + + # Обработка навигации к следующей странице с защитой от crashes + navigation_success = False try: - with _PageOpWatchdog(_PAGE_NEXT_TIMEOUT_S, f"next page {page_number + 1}"): - next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1) - except (PageOperationTimeoutError, PlaywrightError) as nav_exc: - logger.warning( - "go_to_next_page stalled/failed at page %d (%s) — will reopen", - page_number + 1, nav_exc, - ) - next_ok = False - if not next_ok: + navigation_success = self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1) + except (PlaywrightError, PlaywrightTimeoutError, Exception) as nav_exc: + logger.warning("Exception during page navigation (page %d → %d): %s", page_number, page_number + 1, nav_exc) + navigation_success = False + + if not navigation_success: + if not page_result.next_page_detected: + logger.info( + "Failed to navigate to page %d with no next-page control detected — treating pagination as exhausted", + page_number + 1, + ) + break logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1) - try: - page.close() - except Exception: - pass + self._emit_progress("listing_next_page_failed", from_page=page_number, to_page=page_number + 1) + page.close() page = None try: page, _ = self._reopen_listing_and_resume( @@ -919,47 +931,30 @@ class IAAIScraper: listing_url=listing_url, year_min=year_min, year_max=year_max, - max_nav_pages=max(page_number + 1, 500), ) - pages_since_rotation = 0 except ListingResumeError as resume_exc: logger.warning( "Cannot resume at page %d (%s) — treating pagination as exhausted", page_number + 1, resume_exc, ) + pagination_interrupted = True break + else: + self._emit_progress("listing_next_page_done", from_page=page_number, to_page=page_number + 1) if pending_urls: - _process_pending_batch(pending_urls, cars_upserted + cars_failed) - - # Финальный drain последнего in-flight батча. - _drain_pending_future() - - # Обрабатываем отложенные browser fallback'и (Playwright теперь свободен). - if deferred_fallbacks: - fb_pages = min(len(deferred_fallbacks), self._FALLBACK_PARALLEL_PAGES) - logger.info( - "Processing %d deferred browser fallbacks (%d parallel pages)", - len(deferred_fallbacks), fb_pages, + self._emit_progress( + "listing_final_batch_start", + pending_urls=len(pending_urls), + batch_first_url=pending_urls[0] if pending_urls else None, + ) + _process_pending_batch(pending_urls, cars_upserted + cars_failed) + self._emit_progress( + "listing_final_batch_done", + pending_urls=0, + cars_upserted=cars_upserted, + cars_failed=cars_failed, ) - try: - fb_results = self._browser_fallback_parallel( - deferred_fallbacks, len(deferred_fallbacks), fb_pages, - ) - fb_records = fb_results.get("records", []) - if fb_records: - try: - upsert_result = self.persistence.upsert_cars_batch(fb_records) - cars_upserted += upsert_result["inserted"] + upsert_result["updated"] - images_upserted += upsert_result["images_upserted"] - except Exception as exc: - logger.error("Deferred fallback upsert failed: %s", exc) - cars_failed += len(fb_records) - cars_failed += fb_results.get("cars_failed", 0) - failures.extend(fb_results.get("failures", [])) - except Exception as fb_exc: - logger.error("Deferred fallback batch failed: %s", fb_exc) - cars_failed += len(deferred_fallbacks) logger.warning( "sync_listing final progress: pages=%d, discovered=%d, upserted=%d, failed=%d", @@ -969,12 +964,6 @@ class IAAIScraper: cars_failed, ) finally: - # Ensure no future is left dangling on exception path. - try: - _drain_pending_future() - except Exception: - pass - batch_executor.shutdown(wait=True) if page is not None: page.close() @@ -988,6 +977,7 @@ class IAAIScraper: "vehicle_urls": all_raw_urls, "early_stopped": early_stopped, "truncated_by_time_budget": truncated_by_time_budget, + "pagination_interrupted": pagination_interrupted, "pages": pages_info, }, "total": total, @@ -1229,8 +1219,7 @@ class IAAIScraper: """Извлекает IAAI inventoryView.attributes из HTML без браузера. Использует json.JSONDecoder.raw_decode для быстрого поиска JSON - вместо посимвольного сканирования скобок. Если получившийся срез - целиком отдаётся декодеру — пробуем orjson (3-5× быстрее stdlib). + вместо посимвольного сканирования скобок. """ search = "inventoryView" pos = html.find(search) @@ -1247,24 +1236,11 @@ class IAAIScraper: if content_start < 0: return None - # Быстрый путь: вырезаем всё до , пытаемся orjson; иначе stdlib. - obj: dict | None = None - if _HAS_ORJSON: - script_end = html.find("", content_start) - if script_end > content_start: - # Подрезаем хвост до последней закрывающей скобки JSON. - snippet = html[content_start:script_end].rstrip().rstrip(";") - try: - obj = _orjson.loads(snippet) - except Exception: - obj = None - - if obj is None: - decoder = json.JSONDecoder() - try: - obj, _ = decoder.raw_decode(html, content_start) - except (json.JSONDecodeError, ValueError): - return None + decoder = json.JSONDecoder() + try: + obj, _ = decoder.raw_decode(html, content_start) + except (json.JSONDecodeError, ValueError): + return None iv = obj.get("inventoryView") if not iv or not iv.get("attributes"): @@ -1375,6 +1351,26 @@ class IAAIScraper: ] return "; ".join(pairs) + def _ensure_http_fast_path_session(self) -> None: + if self.context is not None: + try: + if self._cookie_header_for_context(): + return + except Exception: + logger.debug("Failed to inspect existing context cookies", exc_info=True) + + warmup_page: Page | None = None + try: + warmup_page = self._get_page_with_warmup() + except Exception as exc: + logger.warning("HTTP fast-path warmup failed: %s", exc) + finally: + if warmup_page is not None: + try: + warmup_page.close() + except Exception: + pass + def _scrape_via_raw_http( self, vehicle_url: str, @@ -1382,19 +1378,11 @@ class IAAIScraper: cookie_header: str, user_agent: str, ) -> CarRecord: - """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive). - - Заголовок Accept-Encoding включает gzip/deflate/br — IAAI отдаёт HTML - со сжатием 5-10×, что снижает байты по сети и время декода. - urllib3 сам прозрачно распаковывает gzip и deflate; для br нужен - пакет `brotli` (он в зависимостях). Если brotli не установлен — он - просто не появится в Accept-Encoding и сервер ответит gzip. - """ + """Быстрый HTTP-запрос через urllib3 (connection pooling / keep-alive).""" headers = { "User-Agent": user_agent, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9", - "Accept-Encoding": self._ACCEPT_ENCODING, "Cache-Control": "no-cache", "Pragma": "no-cache", "Connection": "keep-alive", @@ -1403,10 +1391,7 @@ class IAAIScraper: if cookie_header: headers["Cookie"] = cookie_header - # decode_content=True — urllib3 сам распакует gzip/deflate/br. - response = self._http_pool.request( - "GET", vehicle_url, headers=headers, decode_content=True, - ) + response = self._http_pool.request("GET", vehicle_url, headers=headers) if response.status >= 400: raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") html = response.data.decode("utf-8", errors="ignore") @@ -1479,11 +1464,8 @@ class IAAIScraper: _HTTP_MAX_RETRIES = 2 # Повторов на URL до перехода в браузер _HTTP_RETRY_DELAYS = (0.4, 1.0) # Задержки между повторами _FALLBACK_PARALLEL_PAGES = 4 # Параллельных вкладок для fallback - _FALLBACK_NAV_TIMEOUT_MS = 8000 # Таймаут навигации fallback - # gzip+deflate: нативно в urllib3 (zlib, ~0 CPU). Brotli отключён намеренно — - # python-brotli декодирует на CPU и под 32 параллельными потоками становится - # узким местом (хуже, чем гонять чуть больше байт по сети). - _ACCEPT_ENCODING = "gzip, deflate" + # _FALLBACK_NAV_TIMEOUT_MS теперь берётся из settings.fallback_navigation_timeout_ms + # (см. использование ниже). Хардкод 8000 убран — слишком мало при rate-limit от Imperva. def _browser_fallback_parallel( self, @@ -1508,6 +1490,12 @@ class IAAIScraper: page: Page | None = None try: for url, global_idx in url_slice: + self._emit_progress( + "browser_fallback_open_start", + vehicle_url=url, + vehicle_index=global_idx, + vehicle_total=total, + ) if page is None: page = self._get_page() if self.settings.block_resources: @@ -1517,7 +1505,7 @@ class IAAIScraper: page.goto( url, wait_until="commit", - timeout=self._FALLBACK_NAV_TIMEOUT_MS, + timeout=self.settings.fallback_navigation_timeout_ms, ) except Exception as exc: if self._is_protection_or_network_error(exc): @@ -1533,6 +1521,12 @@ class IAAIScraper: continue try: + self._emit_progress( + "browser_fallback_parse_start", + vehicle_url=url, + vehicle_index=global_idx, + vehicle_total=total, + ) js_data: dict = {} try: js_data = page.evaluate(self._JS_EXTRACT) or {} @@ -1590,6 +1584,15 @@ class IAAIScraper: record = CarRecord.model_validate(db_record.model_dump(mode="json")) local_records.append(record) + self._emit_progress( + "browser_fallback_parse_done", + vehicle_url=url, + vehicle_index=global_idx, + vehicle_total=total, + brand=record.brand, + model=record.model, + year=record.year, + ) logger.debug( "[%d/%d] Parsed %s %s %s (fallback)", global_idx, total, record.brand, record.model, record.year or "?", @@ -1599,6 +1602,13 @@ class IAAIScraper: local_protection += 1 local_failed += 1 local_failures.append({"vehicle_url": url, "error": str(exc)}) + self._emit_progress( + "browser_fallback_parse_failed", + vehicle_url=url, + vehicle_index=global_idx, + vehicle_total=total, + error=str(exc), + ) logger.error("[%d/%d] Failed %s: %s", global_idx, total, url, exc) finally: if page is not None: @@ -1638,15 +1648,11 @@ class IAAIScraper: "protection_events": protection_events, } - cookie_header_override: str | None = None, def sync_batch( self, vehicle_urls: list[str], lane: str = "iaai_cars", parallel_tabs: int | None = None, - *, - skip_browser_fallback: bool = False, - cookie_header_override: str | None = None, ) -> dict: trace_id = self._new_trace_id("sync-batch") started_at = time.perf_counter() @@ -1657,39 +1663,33 @@ class IAAIScraper: cars_failed = 0 images_upserted = 0 records: list[CarRecord] = [] - # cookie_header_override передаётся из главного потока при pipeline-вызове — - # вычисление cookie_header из background thread'а ломает Playwright sync greenlet. - cookie_header = ( - cookie_header_override - if cookie_header_override is not None - else self._cookie_header_for_context() - + failures: list[dict[str, str]] = [] http_successes = 0 http_fallbacks = 0 protection_events = 0 total = len(vehicle_urls) logger.info("sync_batch: %d vehicles, %d parallel workers", total, num_workers) - - # cookie_header_override передаётся из главного потока при pipeline-вызове — - # вычисление cookie_header из background thread'а ломает Playwright sync greenlet. - cookie_header = ( - cookie_header_override - if cookie_header_override is not None - else self._cookie_header_for_context() + self._emit_progress( + "sync_batch_start", + batch_size=total, + parallel_workers=num_workers, + first_vehicle_url=vehicle_urls[0] if vehicle_urls else None, + last_vehicle_url=vehicle_urls[-1] if vehicle_urls else None, + ) + + self._ensure_http_fast_path_session() + cookie_header = self._cookie_header_for_context() + user_agent = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) " + "Gecko/20100101 Firefox/128.0" ) - # UA должен совпадать с UA Playwright-контекста, иначе Imperva видит - # «cookies от Chrome, ходим под Firefox» — типичный сигнал бота. - user_agent = self.settings.fingerprint.user_agent # ── Фаза 1: HTTP fast-path ── fallback_urls: list[tuple[str, int]] = [] def _fetch_one_with_retry(idx_url: tuple[int, str]) -> tuple[int, CarRecord | Exception]: idx, url = idx_url - # Небольшой анти-бёрст джиттер: разносим старт запросов в пуле, - # чтобы N воркеров не били в одну TLS-/PX-волну. - time.sleep(random.uniform(0.0, 0.15)) last_exc: Exception | None = None for attempt in range(1 + self._HTTP_MAX_RETRIES): try: @@ -1726,30 +1726,40 @@ class IAAIScraper: "HTTP phase done: %d OK, %d fallback (%.1fs)", http_successes, http_fallbacks, time.perf_counter() - started_at, ) + self._emit_progress( + "sync_batch_http_done", + batch_size=total, + http_successes=http_successes, + http_fallbacks=http_fallbacks, + ) # ── Фаза 2: браузерный fallback ── - # При skip_browser_fallback=True (pipeline-режим) возвращаем fallback_urls - # вызывающему — Playwright занят сбором листинга в главном потоке, - # его нельзя трогать из фонового executor'а. - deferred_fallback_urls: list[tuple[str, int]] = [] if fallback_urls: - if skip_browser_fallback: - deferred_fallback_urls = fallback_urls - logger.info( - "Deferring browser fallback: %d/%d URLs (pipeline mode)", - len(fallback_urls), total, - ) - else: - fb_pages = min(len(fallback_urls), self._FALLBACK_PARALLEL_PAGES) - logger.info( - "Fallback browser mode: %d/%d URLs, %d parallel pages", - len(fallback_urls), total, fb_pages, - ) - fb_results = self._browser_fallback_parallel(fallback_urls, total, fb_pages) - records.extend(fb_results["records"]) - cars_failed += fb_results["cars_failed"] - protection_events += fb_results["protection_events"] - failures.extend(fb_results["failures"]) + fallback_pages = max( + 1, + min( + self._FALLBACK_PARALLEL_PAGES, + num_workers, + len(fallback_urls), + ), + ) + logger.info( + "Fallback browser mode: %d/%d URLs, %d page(s)", + len(fallback_urls), total, + fallback_pages, + ) + self._emit_progress( + "sync_batch_fallback_start", + fallback_count=len(fallback_urls), + batch_size=total, + first_fallback_url=fallback_urls[0][0] if fallback_urls else None, + fallback_pages=fallback_pages, + ) + fb_results = self._browser_fallback_parallel(fallback_urls, total, fallback_pages) + records.extend(fb_results["records"]) + cars_failed += fb_results["cars_failed"] + protection_events += fb_results["protection_events"] + failures.extend(fb_results["failures"]) # ── Фаза 3: запись в БД ── if records: @@ -1773,6 +1783,16 @@ class IAAIScraper: cars_failed += len(records) status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + self._emit_progress( + "sync_batch_done", + batch_size=total, + cars_upserted=cars_upserted, + cars_failed=cars_failed, + images_upserted=images_upserted, + http_successes=http_successes, + http_fallbacks=http_fallbacks, + status=status, + ) return { "trace_id": trace_id, "status": status, @@ -1784,7 +1804,6 @@ class IAAIScraper: "protection_events": protection_events, "elapsed_seconds": round(time.perf_counter() - started_at, 3), "failures": failures, - "deferred_fallback_urls": deferred_fallback_urls, } def sync_listing( @@ -1797,6 +1816,7 @@ class IAAIScraper: listing_url: str | None = None, year_min: int | None = None, year_max: int | None = None, + skip_mark_sold: bool = False, ): # runtime_config — дефолты; CLI/API аргументы приоритетнее. rc = self.runtime_config.sync @@ -1823,17 +1843,36 @@ class IAAIScraper: try: effective_only_new = self.settings.sync_only_new if only_new is None else only_new - stream_result = self._sync_listing_streaming( + if self._should_use_sitemap_discovery( make=make, model=model, - lane=lane, - limit=limit, - effective_only_new=effective_only_new, - started_at=started_at, listing_url=listing_url, year_min=year_min, year_max=year_max, - ) + limit=limit, + effective_only_new=effective_only_new, + ): + try: + stream_result = self._sync_listing_via_sitemap( + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + ) + except SitemapDiscoveryError as exc: + logger.error("Sitemap-only full scan failed: %s", exc) + raise + else: + stream_result = self._sync_listing_streaming( + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + started_at=started_at, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) listing = stream_result["listing"] total = stream_result["total"] skipped_existing = stream_result["skipped_existing"] @@ -1851,8 +1890,11 @@ class IAAIScraper: or (limit is not None and limit > 0) or listing.get("early_stopped", False) or listing.get("truncated_by_time_budget", False) + or listing.get("pagination_interrupted", False) ) - if all_listing_origin_urls and not is_partial_scan: + if skip_mark_sold: + logger.debug("Skipping mark_sold: caller requested (parallel segment mode)") + elif all_listing_origin_urls and not is_partial_scan: try: sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls) if sold_count: diff --git a/iaai_scraper/storage/db.py b/iaai_scraper/storage/db.py index 22e6fda..b68143f 100644 --- a/iaai_scraper/storage/db.py +++ b/iaai_scraper/storage/db.py @@ -14,6 +14,13 @@ from .schemas import CarRecord logger = logging.getLogger("iaai_scraper.db") +def get_db_session() -> Session: + """Получить новую сессию базы данных.""" + settings = Settings() + persistence = PersistenceService(settings) + return persistence.session_factory() + + CAR_DB_FIELDS = { col.key for col in Car.__table__.columns if col.key not in ("id",) @@ -517,21 +524,38 @@ class PersistenceService: is_postgres = "postgresql" in self.settings.database.url with self.session_scope() as session: + # Страховка от ложного mark_sold при битом/неполном full-scan: + # если текущий список активных URL аномально мал относительно уже активных машин в БД, + # ничего не помечаем проданным. + active_db_count = int(session.execute( + select(func.count()) + .select_from(Car) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like(f"{lane}:%")) + ).scalar_one() or 0) + + current_active_count = len(active_origin_urls) + if active_db_count >= 1000 and current_active_count < max(500, int(active_db_count * 0.25)): + logger.warning( + "Skipping mark_sold: suspiciously small active set (%d URLs vs %d active in DB)", + current_active_count, + active_db_count, + ) + return 0 + if is_postgres: # Создаём временную таблицу с активными URL. - session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP")) + session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT PRIMARY KEY) ON COMMIT DROP")) session.execute(text("TRUNCATE _active_urls")) - # Вставляем активные URL чанками. + # Вставляем активные URL чанками через executemany. url_list = list(active_origin_urls) for i in range(0, len(url_list), _IN_CHUNK_SIZE): chunk = url_list[i:i + _IN_CHUNK_SIZE] - values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) - params = {f"u{j}": url for j, url in enumerate(chunk)} - session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params) - - # Создаём индекс на временной таблице для ускорения JOIN. - session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)")) + session.execute( + text("INSERT INTO _active_urls (url) VALUES (:url) ON CONFLICT DO NOTHING"), + [{"url": url} for url in chunk], + ) # Массовая пометка проданных в PostgreSQL. result = session.execute(text(""" @@ -543,10 +567,10 @@ class PersistenceService: LEFT JOIN _active_urls a ON c.origin_url = a.url WHERE a.url IS NULL AND c.is_sold = FALSE - AND c.origin_id LIKE 'iaai:%%' + AND c.origin_id LIKE :lane_prefix ) sub WHERE cars.id = sub.id - """)) + """), {"lane_prefix": f"{lane}:%"}) count = result.rowcount or 0 else: # Упрощённый путь для SQLite. @@ -554,7 +578,7 @@ class PersistenceService: update(Car) .where(Car.origin_url.notin_(active_origin_urls)) .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like("iaai:%")) + .where(Car.origin_id.like(f"{lane}:%")) .values(is_sold=True) ) result = session.execute(stmt) @@ -573,4 +597,43 @@ class PersistenceService: result = session.execute( select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000) ) - return {str(row[0]) for row in result if row and row[0]} \ No newline at end of file + return {str(row[0]) for row in result if row and row[0]} + + def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]: + """Возвращает все активные (не sold) origin_url для указанного lane/prefix.""" + with self.session_scope() as session: + result = session.execute( + select(Car.origin_url) + .where(Car.origin_id.like(f"{prefix}%")) + .where(Car.is_sold == False) # noqa: E712 + .execution_options(yield_per=10000) + ) + return {str(row[0]) for row in result if row and row[0]} + + def get_active_origin_urls_batch_for_refresh( + self, + *, + prefix: str = "iaai:", + offset: int = 0, + limit: int = 3000, + ) -> list[str]: + """Возвращает батч активных origin_url для циклического hourly refresh.""" + with self.session_scope() as session: + rows = session.execute( + select(Car.origin_url) + .where(Car.origin_id.like(f"{prefix}%")) + .where(Car.is_sold == False) # noqa: E712 + .order_by(Car.last_seen_at.asc(), Car.id.asc()) + .offset(max(0, int(offset))) + .limit(max(1, int(limit))) + ).all() + return [str(row[0]) for row in rows if row and row[0]] + + def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int: + with self.session_scope() as session: + return int(session.execute( + select(func.count()) + .select_from(Car) + .where(Car.origin_id.like(f"{prefix}%")) + .where(Car.is_sold == False) # noqa: E712 + ).scalar_one() or 0) \ No newline at end of file diff --git a/iaai_scraper/storage/models.py b/iaai_scraper/storage/models.py index 29d5566..b9168d4 100644 --- a/iaai_scraper/storage/models.py +++ b/iaai_scraper/storage/models.py @@ -80,3 +80,65 @@ class SyncRun(Base): cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) + + +class VehicleCandidate(Base): + __tablename__ = "vehicle_candidates" + __table_args__ = ( + Index("ix_vehicle_candidates_url", "url"), + Index("ix_vehicle_candidates_status_discovered", "status", "discovered_at"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + url: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) + discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) + status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True) # pending, processing, processed, failed + priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + raw_snapshots: Mapped[list["VehicleRawSnapshot"]] = relationship("VehicleRawSnapshot", back_populates="candidate", cascade="all, delete-orphan") + + +class VehicleRawSnapshot(Base): + __tablename__ = "vehicle_raw_snapshots" + __table_args__ = ( + Index("ix_vehicle_raw_snapshots_candidate_id", "candidate_id"), + Index("ix_vehicle_raw_snapshots_captured_at", "captured_at"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True) + candidate: Mapped[VehicleCandidate] = relationship("VehicleCandidate", back_populates="raw_snapshots") + captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) + method: Mapped[str] = mapped_column(String(20), nullable=False) # http, browser + success: Mapped[bool] = mapped_column(Boolean, nullable=False) + raw_data: Mapped[str | None] = mapped_column(Text, nullable=True) # HTML or JSON content + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + parse_results: Mapped[list["VehicleParseResult"]] = relationship("VehicleParseResult", back_populates="snapshot", cascade="all, delete-orphan") + + +class VehicleParseResult(Base): + __tablename__ = "vehicle_parse_results" + __table_args__ = ( + Index("ix_vehicle_parse_results_snapshot_id", "snapshot_id"), + Index("ix_vehicle_parse_results_parsed_at", "parsed_at"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + snapshot_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True) + snapshot: Mapped[VehicleRawSnapshot] = relationship("VehicleRawSnapshot", back_populates="parse_results") + parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) + success: Mapped[bool] = mapped_column(Boolean, nullable=False) + parsed_data: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON with parsed vehicle data + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) + + +class VehicleRetryQueue(Base): + __tablename__ = "vehicle_retry_queue" + __table_args__ = ( + Index("ix_vehicle_retry_queue_candidate_id", "candidate_id"), + Index("ix_vehicle_retry_queue_retry_at", "retry_at"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True) + reason: Mapped[str] = mapped_column(String(50), nullable=False) # parse_failed, capture_failed, etc. + retry_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3) diff --git a/iaai_scraper/worker/tasks.py b/iaai_scraper/worker/tasks.py index bde465a..581e4f5 100644 --- a/iaai_scraper/worker/tasks.py +++ b/iaai_scraper/worker/tasks.py @@ -1,7 +1,10 @@ # Задачи Celery для синхронизации автомобилей и листинга IAAI. from concurrent.futures import ThreadPoolExecutor +import json import logging +import os +import signal from threading import Event, Thread import time import uuid @@ -13,6 +16,7 @@ from redis import Redis from ..core.config import Settings, parse_listing_segments from ..scraper import IAAIScraper from ..storage.db import PersistenceService +from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats logger = logging.getLogger("iaai_scraper.worker.tasks") @@ -25,6 +29,13 @@ SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3 SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60 SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending" SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task" +SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}" +SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress" +SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total" +SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60 +TASK_PROGRESS_KEY_FMT = "iaai:state:task_progress:{task_id}" +SITEMAP_HOURLY_LAST_COUNT_KEY = "iaai:state:sitemap_hourly_last_count" +SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "iaai:state:sitemap_hourly_refresh_offset" def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0): @@ -91,6 +102,283 @@ def _sync_listing_lock_ttl_seconds() -> int: return max(effective_hard + 120, 300) +def _sync_segment_lock_ttl_seconds() -> int: + return 300 + + +def _task_progress_key(task_id: str) -> str: + return TASK_PROGRESS_KEY_FMT.format(task_id=task_id) + + +def _update_task_progress( + redis_client: Redis, + *, + task_id: str, + stage: str, + ttl_seconds: int, + **payload, +) -> None: + try: + data = { + "task_id": task_id, + "stage": stage, + "ts": int(time.time()), + **payload, + } + redis_client.set( + _task_progress_key(task_id), + json.dumps(data, ensure_ascii=False), + ex=max(60, int(ttl_seconds)), + ) + except Exception: + logger.warning("Failed to update task progress for %s", task_id, exc_info=True) + + +def _clear_task_progress(redis_client: Redis, task_id: str) -> None: + try: + redis_client.delete(_task_progress_key(task_id)) + except Exception: + logger.warning("Failed to clear task progress for %s", task_id, exc_info=True) + + +def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:") + + new_urls = [url for url in discovered_urls if url not in existing_urls] + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai") + + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if new_urls: + with IAAIScraper(settings) as scraper: + batch_size = settings.celery.batch_size + for batch_start in range(0, len(new_urls), batch_size): + batch_urls = new_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": len(discovered_urls) - len(new_urls), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_diff", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "sold_marked": sold_count, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: bool | None) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai") + + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + with IAAIScraper(settings) as scraper: + batch_size = settings.celery.batch_size + for batch_start in range(0, len(discovered_urls), batch_size): + batch_urls = discovered_urls[batch_start:batch_start + batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": 0, + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_full_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": None, + "sold_marked": sold_count, + "transport": discovery_result.stats.transport, + } + + +def _hourly_sitemap_rolling_refresh_sync( + *, + redis_client: Redis, + lane: str, + limit: int | None, + only_new: bool | None, +) -> dict[str, object]: + del limit, only_new + settings = Settings() + persistence = _get_persistence() + persistence.create_tables() + + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) + discovered_urls = discovery_result.vehicle_urls + active_urls = set(discovered_urls) + sold_count = 0 + if active_urls: + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai") + + existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:") + new_urls = [url for url in discovered_urls if url not in existing_urls] + + total_active = persistence.count_active_cars_for_lane("iaai:") + batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size)) + try: + offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0) + except Exception: + offset = 0 + + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix="iaai:", + offset=offset, + limit=batch_size, + ) + if not refresh_urls and total_active > 0: + offset = 0 + refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( + prefix="iaai:", + offset=0, + limit=batch_size, + ) + + next_offset = 0 + if total_active > 0: + next_offset = offset + len(refresh_urls) + if next_offset >= total_active: + next_offset = 0 + + try: + redis_client.set(SITEMAP_HOURLY_REFRESH_OFFSET_KEY, str(next_offset)) + except Exception: + logger.warning("Failed to persist hourly rolling refresh offset", exc_info=True) + + seen: set[str] = set() + target_urls: list[str] = [] + for url in new_urls + refresh_urls: + if url in seen: + continue + seen.add(url) + target_urls.append(url) + + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + + if target_urls: + with IAAIScraper(settings) as scraper: + worker_batch_size = settings.celery.batch_size + for batch_start in range(0, len(target_urls), worker_batch_size): + batch_urls = target_urls[batch_start:batch_start + worker_batch_size] + batch_result = scraper.sync_batch(batch_urls, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + + status = "success" if not failures else ("partial_success" if cars_upserted else "failed") + return { + "status": status, + "run_id": None, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "skipped_existing": max(0, len(discovered_urls) - len(new_urls)), + "elapsed_seconds": None, + "failures": failures, + "full_scan_completed": True, + "hourly_mode": "sitemap_rolling_refresh", + "discovered_urls": len(discovered_urls), + "new_urls": len(new_urls), + "refresh_urls": len(refresh_urls), + "sold_marked": sold_count, + "transport": discovery_result.stats.transport, + "refresh_offset": offset, + "refresh_next_offset": next_offset, + "active_total": total_active, + } + + +def _start_stall_watchdog( + redis_client: Redis, + *, + task_id: str, + stall_timeout_seconds: int, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, stall_timeout_seconds / 3)) + + def _watchdog() -> None: + key = _task_progress_key(task_id) + while not stop_event.wait(interval_seconds): + try: + raw = redis_client.get(key) + if not raw: + continue + data = json.loads(raw) + last_ts = int(data.get("ts") or 0) + if not last_ts: + continue + age = int(time.time()) - last_ts + if age < stall_timeout_seconds: + continue + logger.error( + "Task %s stalled for %ss at stage=%s payload=%s; killing worker process for redelivery", + task_id, + age, + data.get("stage"), + data, + ) + except Exception: + logger.warning("Failed to inspect task progress for stall watchdog", exc_info=True) + continue + os.kill(os.getpid(), signal.SIGKILL) + + thread = Thread(target=_watchdog, name=f"task-stall-watchdog-{task_id[:8]}", daemon=True) + thread.start() + return stop_event, thread + + def _get_persistence() -> PersistenceService: settings = Settings() persistence = PersistenceService(settings) @@ -360,6 +648,204 @@ def _start_lock_heartbeat( return stop_event, thread +def _reset_segments_progress(redis_client: Redis, total: int) -> None: + try: + pipe = redis_client.pipeline() + pipe.delete(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.set(SYNC_SEGMENTS_TOTAL_KEY, str(int(total)), ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.execute() + except Exception: + logger.warning("Failed to reset segments progress", exc_info=True) + + +def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[int, int]: + """Помечает сегмент завершённым. Возвращает (completed_count, total).""" + try: + pipe = redis_client.pipeline() + pipe.sadd(SYNC_SEGMENTS_PROGRESS_KEY, str(int(segment_index))) + pipe.expire(SYNC_SEGMENTS_PROGRESS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS) + pipe.scard(SYNC_SEGMENTS_PROGRESS_KEY) + pipe.get(SYNC_SEGMENTS_TOTAL_KEY) + results = pipe.execute() + completed = int(results[2] or 0) + total = int(results[3] or 0) if results[3] else 0 + return completed, total + except Exception: + logger.warning("Failed to mark segment %d completed", segment_index, exc_info=True) + return 0, 0 + + +@shared_task( + name="iaai_scraper.worker.tasks.sync_segment_task", + bind=True, + max_retries=2, + default_retry_delay=60, + acks_late=True, +) +def sync_segment_task( + self, + segment_index: int, + segment: dict, + lane: str = "iaai_cars", + only_new: bool | None = None, + is_bootstrap: bool = False, +): + """Обработка одного сегмента листинга. Запускается параллельно несколькими воркерами.""" + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + redis_client = _get_redis() + settings = Settings() + + # Per-segment lock — защита от случайного дубля + seg_lock_key = SYNC_SEGMENT_LOCK_KEY_FMT.format(idx=int(segment_index)) + owner_token = f"{task_id}:{uuid.uuid4().hex}" + lock_ttl = _sync_segment_lock_ttl_seconds() + lock_acquired = _acquire_lock(redis_client, seg_lock_key, owner_token, lock_ttl) + if not lock_acquired: + logger.info("sync_segment_task[%d] skipped: already running", segment_index) + return {"status": "skipped", "segment_index": segment_index, "reason": "duplicate"} + + seg_make = segment.get("make") + seg_year_min = segment.get("year_min") + seg_year_max = segment.get("year_max") + seg_label = f"{seg_make or 'ALL'}" + if seg_year_min is not None or seg_year_max is not None: + seg_label += f" ({seg_year_min}-{seg_year_max})" + + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + watchdog_stop: Event | None = None + watchdog_thread: Thread | None = None + stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds)) + progress_ttl = max(lock_ttl + 120, stall_timeout + 120) + + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_started", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(redis_client, seg_lock_key, owner_token, lock_ttl) + watchdog_stop, watchdog_thread = _start_stall_watchdog( + redis_client, + task_id=task_id, + stall_timeout_seconds=stall_timeout, + ) + + try: + def _job(): + with IAAIScraper() as scraper: + scraper.set_progress_callback( + lambda stage, meta: _update_task_progress( + redis_client, + task_id=task_id, + stage=stage, + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + **meta, + ) + ) + base_url = scraper.settings.listing.cars_url + seg_url = scraper._build_segment_listing_url(base_url, seg_make) if seg_make else None + return scraper.sync_listing( + make=None if seg_url else seg_make, + model=None, + lane=lane, + only_new=only_new, + listing_url=seg_url, + year_min=seg_year_min, + year_max=seg_year_max, + skip_mark_sold=True, + ) + + result = _run_browser_job(_job) + segment_done = bool(result.get("full_scan_completed", False)) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_completed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + status=result.get("status", "success"), + cars_upserted=result.get("cars_upserted", 0), + cars_failed=result.get("cars_failed", 0), + ) + + if is_bootstrap and segment_done: + completed, total = _mark_segment_completed(redis_client, segment_index) + logger.warning( + "Segment %d (%s) bootstrap done: %d/%d completed", + segment_index, seg_label, completed, total, + ) + if total > 0 and completed >= total: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + logger.warning("All %d segments completed; bootstrap full scan done", total) + + return { + "status": result.get("status", "success"), + "segment_index": segment_index, + "segment_label": seg_label, + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0), + "full_scan_completed": segment_done, + } + + except SoftTimeLimitExceeded: + logger.warning("sync_segment_task[%d] soft timeout — partial progress saved", segment_index) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_soft_timeout", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + ) + return { + "status": "timed_out", + "segment_index": segment_index, + "segment_label": seg_label, + } + except Exception as exc: + logger.error("sync_segment_task[%d] failed: %s — %s", segment_index, seg_label, exc, exc_info=True) + _update_task_progress( + redis_client, + task_id=task_id, + stage="segment_task_failed", + ttl_seconds=progress_ttl, + segment_index=segment_index, + segment_label=seg_label, + error=str(exc), + ) + try: + raise self.retry(exc=exc) + except self.MaxRetriesExceededError: + return { + "status": "failed", + "segment_index": segment_index, + "segment_label": seg_label, + "error": str(exc), + } + finally: + if watchdog_stop is not None: + watchdog_stop.set() + if watchdog_thread is not None: + watchdog_thread.join(timeout=1) + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=1) + _clear_task_progress(redis_client, task_id) + _release_lock_if_owner(redis_client, seg_lock_key, owner_token) + + @shared_task( name="iaai_scraper.worker.tasks.sync_vehicle_task", bind=True, @@ -477,10 +963,20 @@ def sync_listing_task( } try: + settings = Settings() full_scan_done_before_run = _is_full_scan_done(redis_client) force_bootstrap_full_scan = not full_scan_done_before_run effective_limit = None if force_bootstrap_full_scan else limit effective_only_new = False if force_bootstrap_full_scan else only_new + hourly_mode = settings.discovery.hourly_mode.strip().lower() + discovery_mode = settings.discovery.mode.strip().lower() + prefer_sitemap_mainline = ( + make is None + and model is None + and effective_limit is None + and not effective_only_new + ) + use_hourly_sitemap_sync = full_scan_done_before_run and prefer_sitemap_mainline # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. # Используется только во время bootstrap для пропуска уже обработанных сегментов. @@ -503,6 +999,58 @@ def sync_listing_task( effective_limit, ) + if use_hourly_sitemap_sync: + if hourly_mode == "diff": + logger.info("Hourly mode: running sitemap diff sync instead of full listing traversal") + result = _hourly_sitemap_diff_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + ) + elif hourly_mode == "full_refresh": + logger.info("Hourly mode: running sitemap full refresh of all active vehicles") + result = _hourly_sitemap_full_refresh_sync( + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + ) + else: + logger.info("Hourly mode: running sitemap rolling refresh of active vehicles") + result = _hourly_sitemap_rolling_refresh_sync( + redis_client=redis_client, + lane=lane, + limit=effective_limit, + only_new=effective_only_new, + ) + try: + redis_client.set(SITEMAP_HOURLY_LAST_COUNT_KEY, str(result.get("discovered_urls", 0))) + except Exception: + logger.warning("Failed to persist hourly sitemap count", exc_info=True) + + summary = { + "task_id": task_id, + "run_id": result.get("run_id"), + "status": result.get("status", "success"), + "cars_upserted": result.get("cars_upserted", 0), + "cars_failed": result.get("cars_failed", 0), + "images_upserted": result.get("images_upserted", 0), + "skipped_existing": result.get("skipped_existing", 0), + "elapsed_seconds": result.get("elapsed_seconds"), + "failures_count": len(result.get("failures") or []), + "hourly_mode": result.get("hourly_mode"), + "discovered_urls": result.get("discovered_urls", 0), + "new_urls": result.get("new_urls", 0), + "sold_marked": result.get("sold_marked", 0), + } + logger.info( + "sync_listing_task hourly diff completed: status=%s, new=%d, sold=%d, skipped=%d", + summary["status"], + summary["new_urls"], + summary["sold_marked"], + summary["skipped_existing"], + ) + return summary + _clear_followup_pending(redis_client) heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( @@ -514,9 +1062,91 @@ def sync_listing_task( self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) # Определяем сегменты из конфига. - settings = Settings() segments = parse_listing_segments(settings.listing.listing_segments_json) - use_segmented = bool(segments) and make is None and model is None + use_segmented = ( + bool(segments) + and make is None + and model is None + and not prefer_sitemap_mainline + and discovery_mode != "sitemap" + ) + + if prefer_sitemap_mainline: + if discovery_mode != "sitemap": + logger.warning( + "Unfiltered full scan forcing sitemap discovery despite IAAI_DISCOVERY_MODE=%s", + discovery_mode or "unset", + ) + if segments: + logger.info("Ignoring configured listing segments for unfiltered sitemap full scan") + + # --- Параллельный диспатч сегментов: dispatch & exit --- + if use_segmented and settings.celery.parallel_segments: + # Сегменты уже завершённые (для bootstrap resume) пропускаем по Redis SET. + already_completed: set[int] = set() + if force_bootstrap_full_scan: + try: + raw = redis_client.smembers(SYNC_SEGMENTS_PROGRESS_KEY) or set() + already_completed = {int(x) for x in raw if str(x).strip().lstrip("-").isdigit()} + except Exception: + already_completed = set() + + pending = [ + (idx, seg) for idx, seg in enumerate(segments) + if idx not in already_completed + ] + + if not pending: + # Всё уже сделано — фиксируем bootstrap done. + if force_bootstrap_full_scan: + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) + _clear_bootstrap_failure_streak(redis_client) + logger.warning("Parallel segments: nothing to dispatch (all completed)") + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": 0, + "segments_already_completed": len(already_completed), + } + + # При первом запуске bootstrap фиксируем total, чтобы знать когда остановиться. + if force_bootstrap_full_scan and not already_completed: + _reset_segments_progress(redis_client, len(segments)) + + dispatched = 0 + for idx, seg in pending: + try: + self.app.send_task( + "iaai_scraper.worker.tasks.sync_segment_task", + kwargs={ + "segment_index": idx, + "segment": seg, + "lane": lane, + "only_new": effective_only_new, + "is_bootstrap": force_bootstrap_full_scan, + }, + queue="scraping", + ) + dispatched += 1 + except Exception: + logger.warning("Failed to dispatch segment %d", idx, exc_info=True) + + logger.warning( + "Parallel segments dispatched: %d/%d (already_completed=%d, bootstrap=%s)", + dispatched, len(segments), len(already_completed), force_bootstrap_full_scan, + ) + return { + "status": "success", + "task_id": task_id, + "mode": "parallel_segments", + "segments_total": len(segments), + "segments_dispatched": dispatched, + "segments_already_completed": len(already_completed), + } + # --- конец параллельной ветки --- resume_from_segment = 0 if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None: @@ -608,10 +1238,7 @@ def sync_listing_task( ) if force_bootstrap_full_scan: _set_full_scan_done(redis_client, False) - # SoftTimeLimit считаем failure для circuit breaker: - # если сегмент систематически не укладывается в time limit, - # нельзя крутить followup бесконечно. - _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=True) + _enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=False) # Partial progress уже записан в БД через finish_sync_run. # Не retry — следующий запуск продолжит обработку по расписанию. return { @@ -646,3 +1273,89 @@ def sync_listing_task( heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) if lock_acquired: _release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token) + + +# Новые задачи для ingestion pipeline + +@shared_task( + name="iaai_scraper.worker.tasks.discover_vehicles_task", + bind=True, + max_retries=2, + default_retry_delay=60, + acks_late=True, +) +def discover_vehicles_task(self, max_urls: int | None = None): + """Задача для обнаружения новых URL автомобилей.""" + from ..discovery_service import DiscoveryService + + try: + discovery = DiscoveryService() + count = discovery.discover_new_vehicles(max_urls=max_urls) + logger.info("discover_vehicles_task completed: %d candidates added", count) + return {"status": "success", "candidates_added": count} + except Exception as exc: + logger.error("discover_vehicles_task failed: %s", exc, exc_info=True) + raise self.retry(exc=exc) + + +@shared_task( + name="iaai_scraper.worker.tasks.fetch_pending_candidates_task", + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def fetch_pending_candidates_task(self, limit: int = 10): + """Задача для захвата данных ожидающих кандидатов.""" + from ..fetch_service import FetchService + + try: + fetch = FetchService() + count = fetch.process_pending_candidates(limit=limit) + logger.info("fetch_pending_candidates_task completed: %d candidates processed", count) + return {"status": "success", "candidates_processed": count} + except Exception as exc: + logger.error("fetch_pending_candidates_task failed: %s", exc, exc_info=True) + raise self.retry(exc=exc) + + +@shared_task( + name="iaai_scraper.worker.tasks.enrich_snapshots_task", + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def enrich_snapshots_task(self, limit: int = 10): + """Задача для парсинга и обогащения snapshots.""" + from ..enrichment_service import EnrichmentService + + try: + enrichment = EnrichmentService() + count = enrichment.process_unparsed_snapshots(limit=limit) + logger.info("enrich_snapshots_task completed: %d snapshots enriched", count) + return {"status": "success", "snapshots_enriched": count} + except Exception as exc: + logger.error("enrich_snapshots_task failed: %s", exc, exc_info=True) + raise self.retry(exc=exc) + + +@shared_task( + name="iaai_scraper.worker.tasks.run_ingestion_pipeline_task", + bind=True, + max_retries=2, + default_retry_delay=120, + acks_late=True, +) +def run_ingestion_pipeline_task(self): + """Задача для запуска полного ingestion pipeline.""" + from ..scheduler_service import SchedulerService + + try: + scheduler = SchedulerService() + scheduler.run_full_pipeline() + logger.info("run_ingestion_pipeline_task completed") + return {"status": "success"} + except Exception as exc: + logger.error("run_ingestion_pipeline_task failed: %s", exc, exc_info=True) + raise self.retry(exc=exc) diff --git a/pyproject.toml b/pyproject.toml index 47a44fb..a183db0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ requires-python = ">=3.11" dependencies = [ "alembic>=1.14.0", "celery>=5.4.0", + "curl-cffi>=0.11.0", "fastapi>=0.115.0", "playwright>=1.53.0", "psycopg2-binary>=2.9.9", diff --git a/tests/test_listing.py b/tests/test_listing.py index 7d8e991..3595d79 100644 --- a/tests/test_listing.py +++ b/tests/test_listing.py @@ -8,26 +8,83 @@ from iaai_scraper.browser.listing import ListingCollector class _FakePage: - def __init__(self, counts: dict[str, int]) -> None: + def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None: self._counts = counts + self._attrs = attrs or {} self._evaluate_result = False self._evaluate_values: list[object] = [] + self._html = "" + self.url = "https://www.iaai.com/Vehiclelisting/Cars" + self._current_page_number: int | None = None + self._vehicle_hrefs: list[str] = [] + self._wait_for_function_error: Exception | None = None + self._clicks: dict[str, int] = {} + self._goto_calls: list[str] = [] class _Locator: - def __init__(self, count_value: int) -> None: + def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None: + self._page = page + self._selector = selector self._count_value = count_value + @property + def first(self) -> "_FakePage._Locator": + return self + def count(self) -> int: return self._count_value - def locator(self, selector: str) -> "_FakePage._Locator": - return _FakePage._Locator(self._counts.get(selector, 0)) + def is_visible(self, timeout: int | None = None) -> bool: + _ = timeout + return False - def evaluate(self, _script: str): + def get_attribute(self, name: str, timeout: int | None = None) -> str | None: + _ = timeout + return self._page._attrs.get(self._selector, {}).get(name) + + def click(self, timeout: int | None = None) -> None: + _ = timeout + self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1 + return None + + class _HtmlLocator: + def __init__(self, html: str) -> None: + self._html = html + + def inner_html(self, timeout: int | None = None) -> str: + _ = timeout + return self._html + + def locator(self, selector: str): + if selector == "html": + return _FakePage._HtmlLocator(self._html) + return _FakePage._Locator(self, selector, self._counts.get(selector, 0)) + + def evaluate(self, script: str, *args): + _ = args if self._evaluate_values: return self._evaluate_values.pop(0) + if "querySelectorAll" in script and "VehicleDetail" in script: + return list(self._vehicle_hrefs) + if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script: + return self._current_page_number if self._current_page_number is not None else self._evaluate_result return self._evaluate_result + def wait_for_selector(self, selector: str, timeout: int | None = None) -> None: + _ = selector, timeout + return None + + def wait_for_function(self, script: str, timeout: int | None = None) -> None: + _ = script, timeout + if self._wait_for_function_error is not None: + raise self._wait_for_function_error + return None + + def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None: + _ = wait_until, timeout + self._goto_calls.append(url) + self.url = url + class TestListingUnit(unittest.TestCase): def test_pagination_detection_and_page_number(self) -> None: @@ -63,6 +120,67 @@ class TestListingUnit(unittest.TestCase): ], ) + def test_has_next_page_from_html(self) -> None: + collector = ListingCollector(Settings(), HumanPacer(Settings())) + html = 'Next' + self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42)) + self.assertFalse(collector._has_next_page_from_html("
done
", current_page_number=42)) + + def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None: + collector = ListingCollector(Settings(), HumanPacer(Settings())) + + url = collector._build_listing_page_url( + "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar", + 44, + ) + + self.assertEqual( + url, + "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44", + ) + + def test_collect_current_page_uses_html_first(self) -> None: + collector = ListingCollector(Settings(), HumanPacer(Settings())) + page = _FakePage({}) + page._html = 'Car 555Next' + + result = collector.collect_current_page(page, page_number=1) + + self.assertEqual(len(result.vehicle_links), 1) + self.assertEqual(result.vehicle_links[0].lot_number, "555") + self.assertTrue(result.next_page_detected) + + def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None: + page = _FakePage({}) + page._wait_for_function_error = RuntimeError("same content") + page._current_page_number = 41 + page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"] + + self.assertFalse( + ListingCollector._wait_for_navigation_result( + page, + "/VehicleDetail/111~US", + 41, + old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"), + ) + ) + + def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None: + collector = ListingCollector(Settings(), HumanPacer(Settings())) + page = _FakePage( + {ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1}, + attrs={ + "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"}, + }, + ) + page._wait_for_function_error = RuntimeError("same content") + page._current_page_number = 41 + page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"] + + self.assertFalse(collector.go_to_next_page(page, expected_page_number=41)) + self.assertGreaterEqual(len(page._goto_calls), 1) + self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 3f4a379..70b8d4e 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -6,6 +6,8 @@ from unittest.mock import MagicMock from iaai_scraper.core.config import Settings from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError +from iaai_scraper.core.exceptions import ListingResumeError +from iaai_scraper.discovery import SitemapDiscoveryError from iaai_scraper.scraper import IAAIScraper from iaai_scraper.storage.schemas import CarRecord @@ -85,6 +87,51 @@ class TestScraperSync(unittest.TestCase): scraper2._sync_listing_streaming.assert_called_once() scraper2.collect_listing.assert_not_called() + def test_full_scan_uses_sitemap_discovery_path(self) -> None: + scraper = self._make_scraper() + scraper.settings.discovery.mode = "listing" + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=88) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0) + scraper._sync_listing_via_sitemap = MagicMock(return_value={ + "listing": { + "vehicles_collected": 123, + "early_stopped": False, + "truncated_by_time_budget": False, + "pagination_interrupted": False, + "source": "sitemap", + }, + "total": 123, + "skipped_existing": 0, + "cars_upserted": 120, + "cars_failed": 3, + "images_upserted": 500, + "failures": [{"vehicle_url": "v", "error": "e"}], + "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"}, + }) + scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used")) + + result = scraper.sync_listing() + + scraper._sync_listing_via_sitemap.assert_called_once() + scraper._sync_listing_streaming.assert_not_called() + self.assertEqual(result["cars_upserted"], 120) + self.assertTrue(result["full_scan_completed"]) + + def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=89) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0) + scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down")) + result = scraper.sync_listing() + + scraper._sync_listing_via_sitemap.assert_called_once() + self.assertEqual(result["status"], "failed") + self.assertEqual(result["cars_upserted"], 0) + def test_segmented_sync_calls_per_segment_and_resumes(self) -> None: scraper = self._make_scraper() scraper.persistence.create_tables = MagicMock() @@ -268,6 +315,126 @@ class TestScraperSync(unittest.TestCase): self.assertEqual(result["cars_upserted"], 1) self.assertEqual(result["total"], 1) + def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None: + scraper = self._make_scraper() + scraper.settings.celery.batch_size = 1000 + + page = MagicMock() + page_result = SimpleNamespace( + page_number=1, + vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")], + next_page_detected=True, + ) + + scraper._get_page_with_warmup = MagicMock(return_value=page) + scraper.listing_collector.open_cars_listing = MagicMock() + scraper.listing_collector.apply_filters = MagicMock(return_value={ + "make": None, + "model": None, + "year_min": None, + "year_max": None, + }) + scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) + scraper.listing_collector.go_to_next_page = MagicMock(return_value=False) + scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"]) + scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed")) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "failures": [], + }) + + result = scraper._sync_listing_streaming( + make=None, + model=None, + lane="iaai_cars", + limit=None, + effective_only_new=False, + started_at=0.0, + listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + ) + + self.assertTrue(result["listing"]["pagination_interrupted"]) + + def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None: + scraper = self._make_scraper() + scraper.settings.celery.batch_size = 1000 + + page = MagicMock() + first_page = SimpleNamespace( + page_number=1, + vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")], + next_page_detected=False, + ) + second_page = SimpleNamespace( + page_number=2, + vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")], + next_page_detected=False, + ) + + scraper._get_page_with_warmup = MagicMock(return_value=page) + scraper.listing_collector.open_cars_listing = MagicMock() + scraper.listing_collector.apply_filters = MagicMock(return_value={ + "make": None, + "model": None, + "year_min": None, + "year_max": None, + }) + scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page]) + scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False]) + scraper._extract_page_urls = MagicMock(side_effect=[ + ["https://www.iaai.com/VehicleDetail/111~US"], + ["https://www.iaai.com/VehicleDetail/222~US"], + ]) + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 2, + "cars_failed": 0, + "images_upserted": 0, + "failures": [], + }) + + result = scraper._sync_listing_streaming( + make=None, + model=None, + lane="iaai_cars", + limit=None, + effective_only_new=False, + started_at=0.0, + listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + ) + + self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2) + scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2) + self.assertEqual(result["listing"]["pages_collected"], 2) + + def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None: + scraper = self._make_scraper() + scraper.persistence.create_tables = MagicMock() + scraper.persistence.start_sync_run = MagicMock(return_value=77) + scraper.persistence.finish_sync_run = MagicMock() + scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock() + scraper._sync_listing_streaming = MagicMock(return_value={ + "listing": { + "vehicles_collected": 10, + "early_stopped": False, + "truncated_by_time_budget": False, + "pagination_interrupted": True, + }, + "total": 10, + "skipped_existing": 0, + "cars_upserted": 10, + "cars_failed": 0, + "images_upserted": 0, + "failures": [], + "all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"}, + }) + + result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA") + + self.assertTrue(result["full_scan_completed"] is False) + scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/tests/test_sitemap_discovery.py b/tests/test_sitemap_discovery.py new file mode 100644 index 0000000..c6f3a2e --- /dev/null +++ b/tests/test_sitemap_discovery.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from iaai_scraper.core.config import Settings +from iaai_scraper.discovery.sitemap import ( + SitemapBlockedError, + SitemapDiscoveryError, + SitemapFetchResult, + _filter_vehicle_urls, + discover_vehicle_urls_from_sitemap, + discover_vehicle_urls_from_sitemap_with_stats, +) + + +class TestSitemapDiscovery(unittest.TestCase): + @staticmethod + def _fetch_result(url: str, payload: bytes, *, source: str = "curl_cffi") -> SitemapFetchResult: + return SitemapFetchResult(url=url, payload=payload, source=source) + + def test_filter_vehicle_urls_dedupes_and_normalizes(self) -> None: + urls = _filter_vehicle_urls([ + "https://www.iaai.com/VehicleDetail/111~US?foo=1", + "https://www.iaai.com/VehicleDetail/111~US?bar=2", + "https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/about", + ]) + + self.assertEqual( + urls, + [ + "https://www.iaai.com/VehicleDetail/111~US", + "https://www.iaai.com/VehicleDetail/222~US", + ], + ) + + def test_discover_vehicle_urls_from_sitemap(self) -> None: + index_xml = b""" + + https://www.iaai.com/sitemap-a.xml + https://www.iaai.com/sitemap-b.xml + https://www.iaai.com/sitemapauctions1.xml + + """ + sitemap_a = b""" + + https://www.iaai.com/VehicleDetail/111~US + https://www.iaai.com/VehicleDetail/222~US?x=1 + + """ + sitemap_b = b""" + + https://www.iaai.com/VehicleDetail/222~US?y=2 + https://www.iaai.com/VehicleDetail/333~US + + """ + + def _fake_fetch(url: str) -> SitemapFetchResult: + if url.endswith("sitemap_index.xml"): + return self._fetch_result(url, index_xml) + if url.endswith("sitemap-a.xml"): + return self._fetch_result(url, sitemap_a) + if url.endswith("sitemap-b.xml"): + return self._fetch_result(url, sitemap_b) + raise AssertionError(f"unexpected url: {url}") + + with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): + result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml") + + self.assertEqual( + result, + [ + "https://www.iaai.com/VehicleDetail/111~US", + "https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/VehicleDetail/333~US", + ], + ) + + def test_discover_vehicle_urls_raises_on_empty_index(self) -> None: + empty_index = b"" + with patch( + "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", + return_value=self._fetch_result("https://www.iaai.com/sitemap_index.xml", empty_index), + ): + with self.assertRaises(SitemapDiscoveryError): + discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml") + + def test_discover_vehicle_urls_skips_malformed_and_non_vehicle_sitemaps(self) -> None: + sitemap_vehicle = b""" + + https://www.iaai.com/VehicleDetail/111~US + + """ + sitemap_non_vehicle = b""" + + https://www.iaai.com/SalesList/111~US/04222026 + + """ + index_xml = b""" + + https://www.iaai.com/sitemap1.xml + https://www.iaai.com/sitemapbranches1.xml + https://www.iaai.com/sitemap2.xml + https://www.iaai.com/sitemap3.xml + + """ + + def _fake_fetch(url: str) -> SitemapFetchResult: + if url.endswith("sitemap_index.xml"): + return self._fetch_result(url, index_xml) + if url.endswith("sitemap1.xml"): + return self._fetch_result(url, sitemap_vehicle) + if url.endswith("sitemap2.xml"): + return self._fetch_result(url, sitemap_non_vehicle) + if url.endswith("sitemap3.xml"): + raise SitemapDiscoveryError("broken sitemap") + raise AssertionError(f"unexpected url: {url}") + + with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): + result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml") + + self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"]) + + def test_discover_vehicle_urls_falls_back_to_regex_loc_extraction(self) -> None: + malformed_index = ( + b"" + b"https://www.iaai.com/sitemap1.xml" + b" + https://www.iaai.com/VehicleDetail/111~US + + """ + + def _fake_fetch(url: str) -> SitemapFetchResult: + if url.endswith("sitemap_index.xml"): + return self._fetch_result(url, malformed_index) + if url.endswith("sitemap1.xml"): + return self._fetch_result(url, sitemap_vehicle) + raise AssertionError(f"unexpected url: {url}") + + with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): + result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml") + + self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"]) + + def test_discovery_uses_direct_probe_when_index_has_no_urls(self) -> None: + index_xml = b"" + sitemap_one = b""" + + https://www.iaai.com/VehicleDetail/111~US + + """ + + settings = Settings() + settings.discovery.sitemap_direct_probe_limit = 2 + settings.discovery.sitemap_direct_probe_stop_after_misses = 1 + + def _fake_fetch(url: str) -> SitemapFetchResult: + if url.endswith("sitemap_index.xml"): + return self._fetch_result(url, index_xml) + if url.endswith("sitemap1.xml"): + return self._fetch_result(url, sitemap_one) + raise SitemapDiscoveryError("not found") + + with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): + result = discover_vehicle_urls_from_sitemap_with_stats( + "https://www.iaai.com/sitemap_index.xml", + settings=settings, + ) + + self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"]) + self.assertEqual(result.stats.direct_probe_hits, 1) + + def test_discovery_stats_report_direct_probe_hits(self) -> None: + index_xml = b"" + sitemap1 = b""" + + https://www.iaai.com/VehicleDetail/111~US + + """ + + def _fake_fetch(url: str): + if url.endswith("sitemap_index.xml"): + return self._fetch_result(url, index_xml) + if url.endswith("sitemap1.xml"): + return self._fetch_result(url, sitemap1) + raise SitemapDiscoveryError("not found") + + with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch): + result = discover_vehicle_urls_from_sitemap_with_stats("https://www.iaai.com/sitemap_index.xml") + + self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"]) + self.assertEqual(result.stats.transport, "curl_cffi") + self.assertEqual(result.stats.direct_probe_hits, 1) + self.assertGreaterEqual(result.stats.direct_probe_misses, 1) + + def test_block_page_raises_discovery_error(self) -> None: + settings = Settings() + settings.discovery.sitemap_direct_probe_limit = 1 + settings.discovery.sitemap_direct_probe_stop_after_misses = 1 + + with patch( + "iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", + side_effect=SitemapBlockedError("Anti-bot page returned for https://www.iaai.com/sitemap_index.xml"), + ): + with self.assertRaises(SitemapDiscoveryError): + discover_vehicle_urls_from_sitemap( + "https://www.iaai.com/sitemap_index.xml", + settings=settings, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py index 2f8c088..378d000 100644 --- a/tests/test_worker_tasks.py +++ b/tests/test_worker_tasks.py @@ -6,7 +6,210 @@ from unittest.mock import MagicMock, patch from iaai_scraper.worker import tasks +def make_settings_stub(*, parallel_segments: bool = False) -> MagicMock: + settings_stub = MagicMock() + settings_stub.celery.parallel_segments = parallel_segments + settings_stub.celery.task_soft_time_limit = 3300 + settings_stub.celery.task_time_limit = 3600 + settings_stub.celery.task_stall_timeout_seconds = 600 + settings_stub.listing.listing_segments_json = "ignored" + settings_stub.discovery.mode = "listing" + settings_stub.discovery.hourly_mode = "rolling_refresh" + return settings_stub + + class TestWorkerTaskLockHelpers(unittest.TestCase): + def test_sync_listing_task_uses_hourly_sitemap_even_when_mode_is_not_sitemap(self) -> None: + settings_stub = make_settings_stub(parallel_segments=False) + settings_stub.discovery.mode = "listing" + settings_stub.discovery.hourly_mode = "rolling_refresh" + + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "Settings", return_value=settings_stub), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={ + "status": "success", + "run_id": None, + "cars_upserted": 25, + "cars_failed": 0, + "images_upserted": 50, + "skipped_existing": 100, + "elapsed_seconds": None, + "failures": [], + "hourly_mode": "sitemap_rolling_refresh", + "discovered_urls": 105, + "new_urls": 5, + "refresh_urls": 20, + "sold_marked": 2, + }) as hourly_sync, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + tasks.sync_listing_task.push_request(id="task-hourly-force-sitemap") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh") + hourly_sync.assert_called_once() + release_lock.assert_called_once() + + def test_sync_listing_task_uses_hourly_sitemap_rolling_refresh_after_bootstrap(self) -> None: + settings_stub = make_settings_stub(parallel_segments=False) + settings_stub.discovery.mode = "sitemap" + settings_stub.discovery.hourly_mode = "rolling_refresh" + + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "Settings", return_value=settings_stub), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={ + "status": "success", + "run_id": None, + "cars_upserted": 25, + "cars_failed": 0, + "images_upserted": 50, + "skipped_existing": 100, + "elapsed_seconds": None, + "failures": [], + "hourly_mode": "sitemap_rolling_refresh", + "discovered_urls": 105, + "new_urls": 5, + "refresh_urls": 20, + "sold_marked": 2, + }) as hourly_sync, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + tasks.sync_listing_task.push_request(id="task-hourly") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh") + hourly_sync.assert_called_once() + release_lock.assert_called_once() + + def test_sync_listing_task_can_use_hourly_sitemap_diff_when_configured(self) -> None: + settings_stub = make_settings_stub(parallel_segments=False) + settings_stub.discovery.mode = "sitemap" + settings_stub.discovery.hourly_mode = "diff" + + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "Settings", return_value=settings_stub), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_hourly_sitemap_diff_sync", return_value={ + "status": "success", + "run_id": None, + "cars_upserted": 5, + "cars_failed": 0, + "images_upserted": 10, + "skipped_existing": 100, + "elapsed_seconds": None, + "failures": [], + "hourly_mode": "sitemap_diff", + "discovered_urls": 105, + "new_urls": 5, + "sold_marked": 2, + }) as hourly_sync, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + tasks.sync_listing_task.push_request(id="task-hourly-diff") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["hourly_mode"], "sitemap_diff") + hourly_sync.assert_called_once() + release_lock.assert_called_once() + + def test_sync_listing_task_forces_sitemap_full_scan_in_bootstrap(self) -> None: + settings_stub = make_settings_stub(parallel_segments=False) + settings_stub.discovery.mode = "listing" + + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "Settings", return_value=settings_stub), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[{"make": "HONDA"}]): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + sync_listing_mock = MagicMock(return_value={ + "run_id": 99, + "status": "success", + "full_scan_completed": True, + "cars_upserted": 10, + "cars_failed": 0, + "images_upserted": 20, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [], + }) + sync_listing_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used")) + scraper = MagicMock() + scraper.sync_listing = sync_listing_mock + scraper.sync_listing_segmented = sync_listing_segmented_mock + scraper_ctx = MagicMock() + scraper_ctx.__enter__.return_value = scraper + scraper_ctx.__exit__.return_value = None + + with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + tasks.sync_listing_task.push_request(id="task-bootstrap-sitemap") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + sync_listing_mock.assert_called_once_with( + make=None, + model=None, + lane="iaai_cars", + limit=None, + only_new=False, + ) + sync_listing_segmented_mock.assert_not_called() + release_lock.assert_called_once() + def test_lock_acquire_refresh_release(self) -> None: redis_client = MagicMock() @@ -175,15 +378,17 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): self.assertIsNone(tasks._load_last_completed_segment(redis_client)) redis_client.delete.assert_not_called() - def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None: + def test_sync_listing_bootstrap_prefers_sitemap_over_segment_resume(self) -> None: segments = [ {"make": "ACURA"}, {"make": "AUDI"}, {"make": "BMW"}, {"make": "EAGLE"}, ] + settings_stub = make_settings_stub() with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "Settings", return_value=settings_stub), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ @@ -199,13 +404,15 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) - sync_segmented_mock = MagicMock(return_value={ + sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used")) + sync_listing_mock = MagicMock(return_value={ "run_id": 11, "status": "success", "full_scan_completed": True, "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], }) scraper_ctx = MagicMock() scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock scraper_ctx.__exit__.return_value = None with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): @@ -216,21 +423,42 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") - # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW). - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2) - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) + sync_listing_mock.assert_called_once_with( + make=None, + model=None, + lane="iaai_cars", + limit=None, + only_new=False, + ) + sync_segmented_mock.assert_not_called() release_lock.assert_called_once() - def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None: + def test_sync_listing_hourly_path_ignores_checkpoint_after_full_scan_completed(self) -> None: segments = [{"make": "ACURA"}, {"make": "AUDI"}] + settings_stub = make_settings_stub() with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "Settings", return_value=settings_stub), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={ + "status": "success", + "run_id": None, + "cars_upserted": 1, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": None, + "failures": [], + "hourly_mode": "sitemap_rolling_refresh", + "discovered_urls": 10, + "new_urls": 1, + "refresh_urls": 1, + "sold_marked": 0, + }) as hourly_sync, \ patch.object(tasks.sync_listing_task, "update_state"), \ patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() @@ -242,32 +470,24 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) - sync_segmented_mock = MagicMock(return_value={ - "run_id": 14, "status": "success", "full_scan_completed": True, - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, - "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], - }) - scraper_ctx = MagicMock() - scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock - scraper_ctx.__exit__.return_value = None - - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): - tasks.sync_listing_task.push_request(id="task-792") - try: - result = tasks.sync_listing_task.run() - finally: - tasks.sync_listing_task.pop_request() + tasks.sync_listing_task.push_request(id="task-792") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) - self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"]) + self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh") + hourly_sync.assert_called_once() clear_checkpoint.assert_called() release_lock.assert_called_once() - def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None: + def test_sync_listing_bootstrap_ignores_beyond_segment_checkpoint(self) -> None: segments = [{"make": "ACURA"}, {"make": "AUDI"}] + settings_stub = make_settings_stub() with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "Settings", return_value=settings_stub), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ @@ -283,13 +503,15 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) - sync_segmented_mock = MagicMock(return_value={ + sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used")) + sync_listing_mock = MagicMock(return_value={ "run_id": 15, "status": "success", "full_scan_completed": True, "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], }) scraper_ctx = MagicMock() scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock + scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock scraper_ctx.__exit__.return_value = None with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): @@ -300,7 +522,14 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") - self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) + sync_listing_mock.assert_called_once_with( + make=None, + model=None, + lane="iaai_cars", + limit=None, + only_new=False, + ) + sync_segmented_mock.assert_not_called() release_lock.assert_called_once() def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None: @@ -377,28 +606,38 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): self.assertTrue(should_enqueue) def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None: + settings_stub = make_settings_stub(parallel_segments=False) with patch.object(tasks, "_get_persistence") as get_persistence, \ - patch.object(tasks, "_get_redis") as get_redis, \ - patch.object(tasks, "_acquire_lock", return_value=True), \ - patch.object(tasks, "_is_full_scan_done", return_value=False), \ - patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ - patch.object(tasks, "_release_lock_if_owner") as release_lock, \ - patch.object(tasks, "_try_set_followup_pending", return_value=True), \ - patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \ - patch.object(tasks, "_clear_followup_pending") as clear_pending, \ - patch.object(tasks.sync_listing_task, "update_state"), \ - patch.object(tasks, "_run_browser_job", return_value={ - "run_id": 99, - "status": "failed", - "full_scan_completed": False, - "cars_upserted": 0, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "elapsed_seconds": 1.0, - "failures": [{"vehicle_url": "listing", "error": "bad resume"}], - "listing": {"vehicles_collected": 0}, - }): + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object(tasks, "Settings", return_value=settings_stub), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_try_set_followup_pending", return_value=True), \ + patch.object( + tasks, + "_bump_bootstrap_failure_streak", + return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False), + ) as bump_streak, \ + patch.object(tasks, "_clear_followup_pending") as clear_pending, \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch.object( + tasks, + "_run_browser_job", + return_value={ + "run_id": 99, + "status": "failed", + "full_scan_completed": False, + "cars_upserted": 0, + "cars_failed": 0, + "images_upserted": 0, + "skipped_existing": 0, + "elapsed_seconds": 1.0, + "failures": [{"vehicle_url": "listing", "error": "bad resume"}], + "listing": {"vehicles_collected": 0}, + }, + ): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None