864 lines
36 KiB
Python
864 lines
36 KiB
Python
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import math
|
|
import re
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import Any, Iterator
|
|
from urllib.parse import quote, urljoin
|
|
|
|
import requests
|
|
from requests.adapters import HTTPAdapter
|
|
|
|
from ..core.config import Settings
|
|
|
|
logger = logging.getLogger("iaai_scraper.fast_client")
|
|
|
|
TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
|
|
CHALLENGE_MARKERS = (
|
|
"_incapsula_resource",
|
|
"incapsula",
|
|
"incident id",
|
|
"request unsuccessful",
|
|
"access denied",
|
|
)
|
|
COOKIE_ACCEPT_SELECTORS = (
|
|
"button:has-text('Accept All')",
|
|
"button:has-text('Accept all')",
|
|
"button:has-text('I Agree')",
|
|
"button:has-text('Agree')",
|
|
"button:has-text('Only necessary')",
|
|
"button:has-text('Только необходимые')",
|
|
"button:has-text('Принять все')",
|
|
"[id*='accept']",
|
|
"[class*='accept']",
|
|
)
|
|
LISTING_MARKER = 'id="GBPSearchQuery"'
|
|
DETAIL_MARKER = 'id="ProductDetailsVM"'
|
|
RESIZER_URL = "https://vis.iaai.com/resizer"
|
|
BRAND_SCOPE_OVERRIDES = {
|
|
# IAAI does not resolve every rare make through /Vehiclelisting/Cars/{make}.
|
|
# CUPRA is available through a saved Search scope URL from the site UI.
|
|
"CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d",
|
|
}
|
|
DEFAULT_USER_AGENT = (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/124.0.0.0 Safari/537.36"
|
|
)
|
|
PLAYWRIGHT_REFRESH_POLLS = 8
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FastListingVehicle:
|
|
inventory_id: str
|
|
tenant: str | None
|
|
auction_id: str | None
|
|
auction_date: str | None
|
|
inventory_status: str | None
|
|
currency: str | None
|
|
timed_auction_closed: bool
|
|
timed_auction_indicator: bool
|
|
prebid_indicator: bool
|
|
buynow_indicator: bool
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FastListingPage:
|
|
vehicles: list[FastListingVehicle]
|
|
result_count: int
|
|
page_size: int
|
|
current_page: int
|
|
gbp_search_query: dict[str, Any]
|
|
|
|
|
|
class HybridSessionAuth:
|
|
"""Requests session with Playwright cookie refresh fallback.
|
|
|
|
Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot
|
|
cookies when IAAI returns a challenge or an expected hidden payload is absent.
|
|
"""
|
|
|
|
def __init__(self, settings: Settings) -> None:
|
|
self._settings = settings
|
|
self._thread_local = threading.local()
|
|
self._lock = threading.Lock()
|
|
self._refresh_lock = threading.Lock()
|
|
self._bootstrap_cookies_loaded = False
|
|
self._anonymous_bootstrap_attempted = False
|
|
self._refresh_generation = 0
|
|
self._latest_refresh_cookies: list[dict[str, Any]] = []
|
|
|
|
def request(
|
|
self,
|
|
method: str,
|
|
url: str,
|
|
*,
|
|
timeout: int,
|
|
retries: int,
|
|
retry_backoff_ms: int,
|
|
headers: dict[str, str] | None = None,
|
|
data: Any | None = None,
|
|
json_body: Any | None = None,
|
|
expected_marker: str | None = None,
|
|
) -> requests.Response:
|
|
session = self._get_session()
|
|
self._ensure_anonymous_session_bootstrap(session=session)
|
|
self._sync_session_with_latest_refresh(session)
|
|
last_error: Exception | None = None
|
|
refresh_attempts = 0
|
|
attempt = 0
|
|
max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts))
|
|
|
|
while attempt <= retries:
|
|
try:
|
|
request_started_at = time.perf_counter()
|
|
if self._settings.scraping_profile.verbose_http_logs:
|
|
logger.debug(
|
|
"HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s",
|
|
method,
|
|
url,
|
|
attempt + 1,
|
|
retries + 1,
|
|
timeout,
|
|
expected_marker,
|
|
)
|
|
response = session.request(
|
|
method=method,
|
|
url=url,
|
|
headers=headers,
|
|
data=data,
|
|
json=json_body,
|
|
timeout=(min(10, max(1, timeout)), max(1, timeout)),
|
|
)
|
|
if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400:
|
|
logger.debug(
|
|
"HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s",
|
|
method,
|
|
url,
|
|
response.status_code,
|
|
time.perf_counter() - request_started_at,
|
|
expected_marker,
|
|
)
|
|
except requests.RequestException as exc:
|
|
last_error = exc
|
|
if attempt >= retries:
|
|
break
|
|
self._sleep_backoff(retry_backoff_ms, attempt)
|
|
attempt += 1
|
|
continue
|
|
|
|
if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries:
|
|
response.close()
|
|
self._sleep_backoff(retry_backoff_ms, attempt)
|
|
attempt += 1
|
|
continue
|
|
|
|
if is_challenge_response(
|
|
status_code=response.status_code,
|
|
body_text=response.text,
|
|
expected_marker=expected_marker,
|
|
):
|
|
response.close()
|
|
if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts:
|
|
raise RuntimeError(
|
|
"IAAI challenge persisted after "
|
|
f"{refresh_attempts} Playwright refresh attempts for url={url}"
|
|
)
|
|
refresh_attempts += 1
|
|
logger.info(
|
|
"Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s",
|
|
url,
|
|
response.status_code,
|
|
expected_marker,
|
|
refresh_attempts,
|
|
max_refresh_attempts,
|
|
)
|
|
self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
|
|
logger.info("Retrying HTTP request after Playwright refresh url=%s", url)
|
|
if refresh_attempts > 1:
|
|
self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
|
|
continue
|
|
|
|
return response
|
|
|
|
if last_error is not None:
|
|
raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error
|
|
raise RuntimeError(f"Request failed url={url} after retries")
|
|
|
|
def persist_storage_state(self) -> None:
|
|
session = self._get_session()
|
|
with self._lock:
|
|
self._save_storage_state(session)
|
|
|
|
def _get_session(self) -> requests.Session:
|
|
session = getattr(self._thread_local, "session", None)
|
|
if session is None:
|
|
session = requests.Session()
|
|
pool_size = max(20, int(self._settings.fetch_concurrency) * 2)
|
|
adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size)
|
|
session.mount("http://", adapter)
|
|
session.mount("https://", adapter)
|
|
session.headers.update(
|
|
{
|
|
"user-agent": DEFAULT_USER_AGENT,
|
|
"accept-language": "en-US,en;q=0.9",
|
|
"cache-control": "no-cache",
|
|
"pragma": "no-cache",
|
|
}
|
|
)
|
|
if self._settings.proxy.enabled:
|
|
proxies = self._settings.proxy.to_requests_proxies()
|
|
if proxies:
|
|
session.proxies.update(proxies)
|
|
with self._lock:
|
|
self._bootstrap_session_cookies(session)
|
|
self._thread_local.session = session
|
|
self._thread_local.session_generation = 0
|
|
self._sync_session_with_latest_refresh(session)
|
|
return session
|
|
|
|
def _sync_session_with_latest_refresh(self, session: requests.Session) -> None:
|
|
with self._lock:
|
|
latest_generation = self._refresh_generation
|
|
session_generation = getattr(self._thread_local, "session_generation", 0)
|
|
if latest_generation <= session_generation or not self._latest_refresh_cookies:
|
|
return
|
|
cookies = list(self._latest_refresh_cookies)
|
|
self._apply_cookies_to_session(session, cookies)
|
|
self._thread_local.session_generation = latest_generation
|
|
|
|
def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None:
|
|
if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled:
|
|
return
|
|
if self._session_has_iaai_cookies(session):
|
|
self._anonymous_bootstrap_attempted = True
|
|
return
|
|
with self._lock:
|
|
if self._anonymous_bootstrap_attempted:
|
|
return
|
|
self._anonymous_bootstrap_attempted = True
|
|
logger.info("No IAAI cookies preloaded. Attempting anonymous session bootstrap via Playwright.")
|
|
try:
|
|
self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
|
|
except Exception as exc:
|
|
logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc)
|
|
|
|
@staticmethod
|
|
def _session_has_iaai_cookies(session: requests.Session) -> bool:
|
|
for item in session.cookies:
|
|
domain = str(getattr(item, "domain", "") or "")
|
|
if not domain or "iaai.com" in domain.lower():
|
|
return True
|
|
return False
|
|
|
|
def _bootstrap_session_cookies(self, session: requests.Session) -> None:
|
|
if self._bootstrap_cookies_loaded:
|
|
return
|
|
self._load_storage_state_cookies(session)
|
|
self._bootstrap_cookies_loaded = True
|
|
|
|
def _load_storage_state_cookies(self, session: requests.Session) -> None:
|
|
tokens_file = self._settings.tokens_file
|
|
if not tokens_file:
|
|
return
|
|
path = __import__("pathlib").Path(tokens_file)
|
|
if not path.exists():
|
|
return
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception as exc:
|
|
logger.warning("Failed to read storage state file '%s': %s", path, exc)
|
|
return
|
|
cookies = payload.get("cookies") if isinstance(payload, dict) else None
|
|
if not isinstance(cookies, list):
|
|
return
|
|
applied = 0
|
|
for item in cookies:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
name = parse_text(item.get("name"))
|
|
value = parse_text(item.get("value"))
|
|
if not name or value is None:
|
|
continue
|
|
domain = parse_text(item.get("domain")) or ".iaai.com"
|
|
cookie_path = parse_text(item.get("path")) or "/"
|
|
expires = parse_int(item.get("expires"))
|
|
session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
|
|
applied += 1
|
|
if applied:
|
|
logger.info("Loaded %s cookies from storage state", applied)
|
|
|
|
def _refresh_session_via_playwright(
|
|
self,
|
|
*,
|
|
expected_marker: str | None = None,
|
|
session: requests.Session | None = None,
|
|
) -> None:
|
|
target_session = session or self._get_session()
|
|
with self._lock:
|
|
baseline_generation = self._refresh_generation
|
|
|
|
with self._refresh_lock:
|
|
with self._lock:
|
|
if self._refresh_generation > baseline_generation and self._latest_refresh_cookies:
|
|
self._apply_cookies_to_session(target_session, self._latest_refresh_cookies)
|
|
self._thread_local.session_generation = self._refresh_generation
|
|
return
|
|
|
|
logger.info("IAAI session challenge detected. Refreshing session via Playwright.")
|
|
cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
|
|
logger.info("Playwright refresh returned %d cookies", len(cookies))
|
|
self._apply_cookies_to_session(target_session, cookies)
|
|
logger.info("Playwright cookies applied to requests session")
|
|
|
|
with self._lock:
|
|
self._refresh_generation += 1
|
|
self._latest_refresh_cookies = list(cookies)
|
|
self._anonymous_bootstrap_attempted = True
|
|
refreshed_generation = self._refresh_generation
|
|
self._thread_local.session_generation = refreshed_generation
|
|
logger.info("Playwright refresh completed generation=%s", refreshed_generation)
|
|
|
|
def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
|
|
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
with sync_playwright() as playwright:
|
|
browser = playwright.chromium.launch(headless=self._settings.headless)
|
|
try:
|
|
context = browser.new_context(
|
|
locale=self._settings.fingerprint.locale,
|
|
viewport={"width": 1366, "height": 768},
|
|
user_agent=DEFAULT_USER_AGENT,
|
|
proxy=self._settings.proxy.to_playwright_dict(),
|
|
)
|
|
page = context.new_page()
|
|
home_target = self._settings.home_url
|
|
filtered_urls = self._settings.listing.filtered_search_urls
|
|
target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars")
|
|
timeout_ms = max(30_000, self._settings.default_timeout_ms)
|
|
logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER)
|
|
page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
|
|
self._accept_cookie_banner(page)
|
|
page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
|
|
self._accept_cookie_banner(page)
|
|
self._wait_until_non_challenge(
|
|
page=page,
|
|
target=target,
|
|
timeout_ms=timeout_ms,
|
|
expected_marker=expected_marker or LISTING_MARKER,
|
|
)
|
|
cookies = context.cookies()
|
|
logger.info("Playwright context returned %d cookies", len(cookies))
|
|
except PlaywrightTimeoutError as exc:
|
|
raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
|
|
finally:
|
|
try:
|
|
browser.close()
|
|
except Exception as exc:
|
|
logger.info("Playwright browser close failed after cookie refresh: %s", exc)
|
|
|
|
if not isinstance(cookies, list) or not cookies:
|
|
raise RuntimeError("Playwright refresh did not return cookies")
|
|
return [cookie for cookie in cookies if isinstance(cookie, dict)]
|
|
|
|
@staticmethod
|
|
def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None:
|
|
session.cookies.clear()
|
|
for cookie in cookies:
|
|
name = parse_text(cookie.get("name"))
|
|
value = parse_text(cookie.get("value"))
|
|
if not name or value is None:
|
|
continue
|
|
domain = parse_text(cookie.get("domain")) or ".iaai.com"
|
|
cookie_path = parse_text(cookie.get("path")) or "/"
|
|
expires = parse_int(cookie.get("expires"))
|
|
session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
|
|
|
|
@staticmethod
|
|
def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None:
|
|
poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
|
|
navigation_error_count = 0
|
|
for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
|
|
try:
|
|
page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
|
|
except Exception:
|
|
pass
|
|
page.wait_for_timeout(poll_ms)
|
|
body: str | None = None
|
|
for _ in range(3):
|
|
try:
|
|
body = page.content()
|
|
break
|
|
except Exception as exc:
|
|
message = str(exc).lower()
|
|
if "page.content" not in message or "navigating and changing the content" not in message:
|
|
raise
|
|
navigation_error_count += 1
|
|
page.wait_for_timeout(max(200, poll_ms // 4))
|
|
if body is not None and not is_challenge_response(
|
|
status_code=200,
|
|
body_text=body,
|
|
expected_marker=expected_marker,
|
|
):
|
|
logger.info(
|
|
"Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s",
|
|
target,
|
|
poll_index + 1,
|
|
PLAYWRIGHT_REFRESH_POLLS,
|
|
expected_marker,
|
|
)
|
|
return
|
|
try:
|
|
logger.info(
|
|
"Playwright session refresh still waiting target=%s poll=%s/%s marker=%s",
|
|
target,
|
|
poll_index + 1,
|
|
PLAYWRIGHT_REFRESH_POLLS,
|
|
expected_marker,
|
|
)
|
|
page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
|
|
except Exception:
|
|
pass
|
|
raise RuntimeError(
|
|
"Playwright refresh completed but challenge page is still active "
|
|
f"(navigation_content_errors={navigation_error_count})"
|
|
)
|
|
|
|
@staticmethod
|
|
def _accept_cookie_banner(page: Any) -> None:
|
|
for selector in COOKIE_ACCEPT_SELECTORS:
|
|
try:
|
|
locator = page.locator(selector).first
|
|
if locator.count() == 0 or not locator.is_visible(timeout=500):
|
|
continue
|
|
locator.click(timeout=2_000)
|
|
page.wait_for_timeout(250)
|
|
return
|
|
except Exception:
|
|
continue
|
|
|
|
def _save_storage_state(self, session: requests.Session) -> None:
|
|
tokens_file = self._settings.tokens_file
|
|
if not tokens_file:
|
|
return
|
|
path = __import__("pathlib").Path(tokens_file)
|
|
cookies: list[dict[str, Any]] = []
|
|
for cookie in session.cookies:
|
|
payload: dict[str, Any] = {
|
|
"name": cookie.name,
|
|
"value": cookie.value,
|
|
"domain": cookie.domain or ".iaai.com",
|
|
"path": cookie.path or "/",
|
|
"httpOnly": False,
|
|
"secure": bool(cookie.secure),
|
|
"sameSite": "Lax",
|
|
}
|
|
if cookie.expires is not None:
|
|
payload["expires"] = int(cookie.expires)
|
|
cookies.append(payload)
|
|
try:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
except PermissionError as exc:
|
|
logger.info("Cannot persist IAAI storage state to '%s': %s", path, exc)
|
|
except OSError as exc:
|
|
logger.info("Failed to persist IAAI storage state to '%s': %s", path, exc)
|
|
|
|
@staticmethod
|
|
def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None:
|
|
if retry_backoff_ms <= 0:
|
|
return
|
|
time.sleep(retry_backoff_ms * (2**attempt) / 1000)
|
|
|
|
|
|
class IAAIFastClient:
|
|
def __init__(self, settings: Settings) -> None:
|
|
self._settings = settings
|
|
self._auth = HybridSessionAuth(settings)
|
|
|
|
def persist_session_state(self) -> None:
|
|
self._auth.persist_storage_state()
|
|
|
|
def iter_listing_vehicles(
|
|
self,
|
|
*,
|
|
listing_start_url: str | None = None,
|
|
make: str | None = None,
|
|
max_pages: int | None = None,
|
|
) -> Iterator[FastListingVehicle]:
|
|
seen_inventory_ids: set[str] = set()
|
|
scope_paths = resolve_listing_scope_paths(
|
|
listing_start_url=listing_start_url or "",
|
|
brands={make} if make else set(),
|
|
)
|
|
for scope_path in scope_paths:
|
|
first_page_html = self._fetch_listing_first_page(scope_path)
|
|
search_scope_path = build_search_scope_path_from_html(first_page_html)
|
|
if search_scope_path and search_scope_path != scope_path:
|
|
logger.info(
|
|
"Resolved listing scope to fast Search URL: scope=%s search_scope=%s",
|
|
scope_path,
|
|
search_scope_path,
|
|
)
|
|
scope_path = search_scope_path
|
|
first_page = parse_listing_page(first_page_html)
|
|
for vehicle in first_page.vehicles:
|
|
if vehicle.inventory_id in seen_inventory_ids:
|
|
continue
|
|
seen_inventory_ids.add(vehicle.inventory_id)
|
|
yield vehicle
|
|
|
|
page_size = max(1, first_page.page_size)
|
|
total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size))
|
|
if max_pages is not None and max_pages > 0:
|
|
total_pages = min(total_pages, max_pages)
|
|
gbp_search_query = first_page.gbp_search_query
|
|
for page_number in range(2, total_pages + 1):
|
|
page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size)
|
|
parsed_page = parse_listing_page(page_html)
|
|
gbp_search_query = parsed_page.gbp_search_query
|
|
for vehicle in parsed_page.vehicles:
|
|
if vehicle.inventory_id in seen_inventory_ids:
|
|
continue
|
|
seen_inventory_ids.add(vehicle.inventory_id)
|
|
yield vehicle
|
|
|
|
def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]:
|
|
escaped_id = quote(inventory_id, safe="~")
|
|
url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
|
|
response = self._auth.request(
|
|
"GET",
|
|
url,
|
|
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
|
|
retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
|
|
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
|
|
headers={"accept": "text/html,application/xhtml+xml"},
|
|
expected_marker=DETAIL_MARKER,
|
|
)
|
|
with response:
|
|
if response.status_code >= 400:
|
|
raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
|
|
return parse_product_details_vm(response.text)
|
|
|
|
def fetch_vehicle_detail_html(self, inventory_id: str) -> str:
|
|
escaped_id = quote(inventory_id, safe="~")
|
|
url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
|
|
response = self._auth.request(
|
|
"GET",
|
|
url,
|
|
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
|
|
retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
|
|
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
|
|
headers={"accept": "text/html,application/xhtml+xml"},
|
|
expected_marker=DETAIL_MARKER,
|
|
)
|
|
with response:
|
|
if response.status_code >= 400:
|
|
raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
|
|
return response.text
|
|
|
|
def _fetch_listing_first_page(self, scope_path: str) -> str:
|
|
url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/"))
|
|
response = self._auth.request(
|
|
"GET",
|
|
url,
|
|
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
|
|
retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
|
|
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
|
|
headers={"accept": "text/html,application/xhtml+xml"},
|
|
expected_marker=LISTING_MARKER,
|
|
)
|
|
with response:
|
|
if response.status_code >= 400:
|
|
raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}")
|
|
return response.text
|
|
|
|
def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str:
|
|
query_payload = dict(gbp_search_query)
|
|
query_payload["CurrentPage"] = page_number
|
|
query_payload["PageSize"] = page_size
|
|
search_url = urljoin(self._settings.home_url, "Search")
|
|
common_headers = {
|
|
"accept": "text/html,application/xhtml+xml,*/*",
|
|
"x-requested-with": "XMLHttpRequest",
|
|
}
|
|
attempts: list[tuple[dict[str, str], Any, Any]] = [
|
|
({**common_headers, "content-type": "application/json"}, None, query_payload),
|
|
({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}),
|
|
({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None),
|
|
({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None),
|
|
]
|
|
attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))]
|
|
last_error: Exception | None = None
|
|
for headers, data, json_body in attempts:
|
|
try:
|
|
response = self._auth.request(
|
|
"POST",
|
|
search_url,
|
|
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
|
|
retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
|
|
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
|
|
headers=headers,
|
|
data=data,
|
|
json_body=json_body,
|
|
expected_marker=LISTING_MARKER,
|
|
)
|
|
with response:
|
|
if response.status_code >= 400:
|
|
raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}")
|
|
body = response.text
|
|
if LISTING_MARKER not in body:
|
|
raise RuntimeError("Listing page response does not include GBPSearchQuery")
|
|
return body
|
|
except Exception as exc:
|
|
last_error = exc
|
|
continue
|
|
if last_error is not None:
|
|
raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error
|
|
raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}")
|
|
|
|
|
|
def build_brand_scope_paths(brands: set[str]) -> list[str]:
|
|
if not brands:
|
|
return ["/Vehiclelisting/Cars"]
|
|
paths: list[str] = []
|
|
for brand in sorted(brands):
|
|
raw = brand.strip()
|
|
if not raw:
|
|
continue
|
|
override = BRAND_SCOPE_OVERRIDES.get(raw.upper())
|
|
if override:
|
|
if override not in paths:
|
|
paths.append(override)
|
|
continue
|
|
slug_hyphen = quote(raw.replace(" ", "-"), safe="-")
|
|
slug_raw = quote(raw, safe="")
|
|
for slug in (slug_hyphen, slug_raw):
|
|
path = f"/Vehiclelisting/Cars/{slug}"
|
|
if path not in paths:
|
|
paths.append(path)
|
|
return paths or ["/Vehiclelisting/Cars"]
|
|
|
|
|
|
def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
|
|
explicit_scope = listing_start_url.strip()
|
|
if explicit_scope:
|
|
if explicit_scope.lower().startswith(("http://", "https://", "/")):
|
|
return [explicit_scope]
|
|
return [f"/Search?url={explicit_scope}"]
|
|
return build_brand_scope_paths(brands)
|
|
|
|
|
|
def build_search_scope_path_from_html(html_text: str) -> str | None:
|
|
tiny_url = parse_attribute_value(html_text, "data-tinyurl")
|
|
if tiny_url:
|
|
return f"/Search?url={tiny_url}"
|
|
|
|
data_query_raw = parse_attribute_value(html_text, "data-query")
|
|
if data_query_raw:
|
|
try:
|
|
data_query = json.loads(data_query_raw)
|
|
except (json.JSONDecodeError, ValueError, TypeError):
|
|
data_query = None
|
|
if isinstance(data_query, dict):
|
|
url_value = parse_text(data_query.get("Url"))
|
|
if url_value:
|
|
return f"/Search?url={url_value}"
|
|
return None
|
|
|
|
|
|
def parse_listing_page(html_text: str) -> FastListingPage:
|
|
gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
|
|
vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
|
|
result_count_raw = parse_hidden_input_value(html_text, "ResultCount")
|
|
page_size_raw = parse_hidden_input_value(html_text, "PageSize")
|
|
current_page_raw = parse_hidden_input_value(html_text, "CurrentPage")
|
|
if not gbp_raw:
|
|
raise RuntimeError("Listing page missing GBPSearchQuery")
|
|
if vehicle_raw is None:
|
|
raise RuntimeError("Listing page missing VehicleDetails")
|
|
gbp_payload = json.loads(gbp_raw)
|
|
if not isinstance(gbp_payload, dict):
|
|
raise RuntimeError("GBPSearchQuery payload is not object")
|
|
vehicle_payload = json.loads(vehicle_raw)
|
|
if not isinstance(vehicle_payload, list):
|
|
raise RuntimeError("VehicleDetails payload is not array")
|
|
|
|
vehicles: list[FastListingVehicle] = []
|
|
for item in vehicle_payload:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
inventory_id = parse_text(item.get("Id"))
|
|
if not inventory_id:
|
|
continue
|
|
vehicles.append(
|
|
FastListingVehicle(
|
|
inventory_id=inventory_id,
|
|
tenant=parse_text(item.get("Tenant")),
|
|
auction_id=parse_text(item.get("ActnLnId")),
|
|
auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")),
|
|
inventory_status=parse_text(item.get("InventoryStatus")),
|
|
currency=parse_text(item.get("Currency")),
|
|
timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")),
|
|
timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")),
|
|
prebid_indicator=parse_bool(item.get("PreBidIndicator")),
|
|
buynow_indicator=parse_bool(item.get("BuyNowIndicator")),
|
|
)
|
|
)
|
|
return FastListingPage(
|
|
vehicles=vehicles,
|
|
result_count=parse_int(result_count_raw) or len(vehicles),
|
|
page_size=parse_int(page_size_raw) or max(1, len(vehicles)),
|
|
current_page=parse_int(current_page_raw) or 1,
|
|
gbp_search_query=gbp_payload,
|
|
)
|
|
|
|
|
|
def parse_product_details_vm(html_text: str) -> dict[str, Any]:
|
|
match = re.search(
|
|
r"<script[^>]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*</script>",
|
|
html_text,
|
|
flags=re.DOTALL | re.IGNORECASE,
|
|
)
|
|
if match is None:
|
|
raise RuntimeError("ProductDetailsVM script not found")
|
|
payload = json.loads(match.group(1))
|
|
if not isinstance(payload, dict):
|
|
raise RuntimeError("ProductDetailsVM root is not object")
|
|
return payload
|
|
|
|
|
|
def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
|
|
escaped_id = re.escape(input_id)
|
|
patterns = (
|
|
rf"<input[^>]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"",
|
|
rf"<input[^>]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'",
|
|
)
|
|
for pattern in patterns:
|
|
match = re.search(pattern, html_text, flags=re.IGNORECASE)
|
|
if match is not None:
|
|
return html.unescape(match.group(1))
|
|
return None
|
|
|
|
|
|
def parse_attribute_value(html_text: str, attribute_name: str) -> str | None:
|
|
escaped_name = re.escape(attribute_name)
|
|
patterns = (
|
|
rf"\b{escaped_name}=\"([^\"]*)\"",
|
|
rf"\b{escaped_name}='([^']*)'",
|
|
)
|
|
for pattern in patterns:
|
|
match = re.search(pattern, html_text, flags=re.IGNORECASE)
|
|
if match is not None:
|
|
value = html.unescape(match.group(1)).strip()
|
|
return value or None
|
|
return None
|
|
|
|
|
|
def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
|
|
seen_fullres: set[str] = set()
|
|
images: list[dict[str, str | int]] = []
|
|
for index, item in enumerate(image_keys):
|
|
if not isinstance(item, dict):
|
|
continue
|
|
key = parse_text(item.get("k"))
|
|
if key is None:
|
|
continue
|
|
width = parse_int(item.get("w")) or 1600
|
|
height = parse_int(item.get("h")) or 1200
|
|
if width <= 0:
|
|
width = 1600
|
|
if height <= 0:
|
|
height = 1200
|
|
order_index = parse_int(item.get("i"))
|
|
if order_index is None:
|
|
order_index = parse_int(item.get("in"))
|
|
if order_index is None:
|
|
order_index = index
|
|
preview_width = min(640, width)
|
|
preview_height = max(1, int(round(height * (preview_width / width))))
|
|
escaped_key = quote(key, safe="~")
|
|
fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}"
|
|
preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}"
|
|
if fullres in seen_fullres:
|
|
continue
|
|
seen_fullres.add(fullres)
|
|
images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview})
|
|
images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image"))))
|
|
return images
|
|
|
|
|
|
def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool:
|
|
if status_code in {401, 403}:
|
|
return True
|
|
if _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
|
|
return False
|
|
lowered = (body_text or "").lower()
|
|
if any(marker in lowered for marker in CHALLENGE_MARKERS):
|
|
return True
|
|
if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
|
|
if "<html" in lowered or "<body" in lowered:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _expected_marker_present(*, body_text: str, expected_marker: str | None) -> bool:
|
|
if not expected_marker:
|
|
return False
|
|
if expected_marker in body_text:
|
|
return True
|
|
if '"' in expected_marker and expected_marker.replace('"', "'") in body_text:
|
|
return True
|
|
if "'" in expected_marker and expected_marker.replace("'", '"') in body_text:
|
|
return True
|
|
marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker)
|
|
if marker_match is None:
|
|
return False
|
|
marker_id = re.escape(marker_match.group(1))
|
|
return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE))
|
|
|
|
|
|
def parse_text(value: Any) -> str | None:
|
|
if isinstance(value, str):
|
|
text = value.strip()
|
|
return text if text else None
|
|
return None
|
|
|
|
|
|
def parse_bool(value: Any) -> bool:
|
|
if isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, str):
|
|
return value.strip().lower() in {"true", "1", "yes", "on"}
|
|
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
return value != 0
|
|
return False
|
|
|
|
|
|
def parse_int(value: Any) -> int | None:
|
|
if value is None or isinstance(value, bool):
|
|
return None
|
|
if isinstance(value, int):
|
|
return value
|
|
if isinstance(value, float):
|
|
return int(round(value))
|
|
if isinstance(value, str):
|
|
text = value.strip()
|
|
if not text:
|
|
return None
|
|
normalized = text.replace(",", "").replace(" ", "").replace("$", "")
|
|
match = re.search(r"-?\d+(?:\.\d+)?", normalized)
|
|
if match is None:
|
|
return None
|
|
try:
|
|
return int(round(float(match.group(0))))
|
|
except ValueError:
|
|
return None
|
|
return None
|