from __future__ import annotations import gzip import io import logging import random import re import time import xml.etree.ElementTree as ET from dataclasses import dataclass from typing import Iterable from urllib.error import HTTPError, URLError from urllib.parse import urljoin, urlsplit, urlunsplit from urllib.request import Request, urlopen from ..core.config import Settings try: from curl_cffi import requests as curl_requests except ImportError: # pragma: no cover - optional runtime dependency guard curl_requests = None logger = logging.getLogger("iaai_scraper.discovery.sitemap") DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" _LOC_TAG_RE = re.compile(rb"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) _XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL) _BLOCK_MARKERS = ( b"pardon our interruption", b"incapsula", b"please stand by", b"_incapsula_resource", b"as you were browsing something about your browser", ) _HTML_MARKERS = (b" None: self.settings = settings self.discovery = settings.discovery self.proxy_url = settings.proxy.server self.proxy_auth = None if settings.proxy.username: password = settings.proxy.password or "" self.proxy_auth = (settings.proxy.username, password) self._transport_name = self._resolve_transport_name() @property def transport_name(self) -> str: return self._transport_name def _resolve_transport_name(self) -> str: if self.discovery.sitemap_use_curl_cffi and curl_requests is not None: return "curl_cffi" return "urllib" def fetch(self, url: str) -> SitemapFetchResult: attempts = max(1, int(self.discovery.sitemap_retry_attempts)) last_exc: Exception | None = None for attempt in range(1, attempts + 1): try: result = self._fetch_once(url) if result.blocked: raise SitemapBlockedError(f"Anti-bot page returned for {url}") return result except SitemapBlockedError as exc: last_exc = exc logger.warning( "Sitemap fetch blocked (attempt %d/%d, transport=%s): %s", attempt, attempts, self.transport_name, url, ) except Exception as exc: last_exc = exc logger.warning( "Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s", attempt, attempts, self.transport_name, url, exc, ) if attempt >= attempts: break time.sleep(self._backoff_delay(attempt)) if last_exc is None: raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}") if isinstance(last_exc, SitemapDiscoveryError): raise last_exc raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc def _fetch_once(self, url: str) -> SitemapFetchResult: if self.transport_name == "curl_cffi": return self._fetch_with_curl_cffi(url) return self._fetch_with_urllib(url) def _build_headers(self) -> dict[str, str]: headers = dict(_DEFAULT_HEADERS) headers["User-Agent"] = self.settings.fingerprint.user_agent return headers def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult: assert curl_requests is not None response = curl_requests.get( url, headers=self._build_headers(), timeout=self.discovery.sitemap_timeout_seconds, impersonate=random.choice(_CURL_IMPERSONATE_CHOICES), proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None, proxy_auth=self.proxy_auth, allow_redirects=True, ) payload = self._decode_payload( payload=response.content, encoding=(response.headers.get("Content-Encoding") or ""), url=url, ) blocked = _looks_like_block_page(payload) return SitemapFetchResult( url=url, payload=payload, source="curl_cffi", blocked=blocked, status_code=int(response.status_code), content_type=response.headers.get("Content-Type"), ) def _fetch_with_urllib(self, url: str) -> SitemapFetchResult: request = Request(url, headers=self._build_headers()) try: with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response: payload = response.read() decoded = self._decode_payload( payload=payload, encoding=(response.headers.get("Content-Encoding") or ""), url=url, ) return SitemapFetchResult( url=url, payload=decoded, source="urllib", blocked=_looks_like_block_page(decoded), status_code=getattr(response, "status", None), content_type=response.headers.get("Content-Type"), ) except HTTPError as exc: payload = exc.read() if hasattr(exc, "read") else b"" decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url) blocked = exc.code in {403, 429} or _looks_like_block_page(decoded) if blocked: raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc except URLError as exc: raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc @staticmethod def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes: normalized = (encoding or "").lower().strip() if normalized == "gzip" or url.lower().endswith(".gz"): try: return gzip.GzipFile(fileobj=io.BytesIO(payload)).read() except OSError: return payload return payload def _backoff_delay(self, attempt: int) -> float: base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds)) jitter = random.uniform(0.05, 0.35) return base * (2 ** (attempt - 1)) + jitter def _is_vehicle_sitemap_url(url: str) -> bool: lowered = url.strip().lower() if "sitemapbranches" in lowered or "sitemapauctions" in lowered: return False return lowered.endswith(".xml") or lowered.endswith(".xml.gz") def _normalize_vehicle_url(url: str) -> str: parts = urlsplit(url.strip()) return urlunsplit((parts.scheme, parts.netloc, parts.path, "", "")) def _local_name(tag: str) -> str: if "}" in tag: return tag.rsplit("}", 1)[1] return tag def _looks_like_block_page(payload: bytes) -> bool: sample = payload[:8192].lower() if any(marker in sample for marker in _BLOCK_MARKERS): return True if any(marker in sample for marker in _HTML_MARKERS) and b"" not in sample: return True return False def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]: for match in _LOC_TAG_RE.finditer(xml_bytes): try: value = match.group(1).decode("utf-8", errors="ignore").strip() except Exception: continue if value: yield value def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]: if _looks_like_block_page(xml_bytes): raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML") if not _XML_PREFIX_RE.search(xml_bytes[:256]): fallback_values = list(_iter_loc_values_fallback(xml_bytes)) if fallback_values: yield from fallback_values return try: root = ET.fromstring(xml_bytes) except ET.ParseError as exc: fallback_values = list(_iter_loc_values_fallback(xml_bytes)) if fallback_values: logger.warning( "Falling back to regex sitemap loc extraction after XML parse error: %s", exc, ) yield from fallback_values return raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc for element in root.iter(): if _local_name(element.tag) != "loc": continue if not element.text: continue value = element.text.strip() if value: yield value def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]: result: list[str] = [] seen: set[str] = set() for url in urls: normalized = _normalize_vehicle_url(url) if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized: continue if normalized in seen: continue seen.add(normalized) result.append(normalized) return result def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool: return any("/vehicledetail/" in url.lower() for url in urls) def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]: base_dir = index_url.rsplit("/", 1)[0] + "/" return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)] def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]: logger.info("Downloading sitemap index: %s", index_url) try: index_result = downloader.fetch(index_url) sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)] if sitemap_urls: return sitemap_urls except SitemapBlockedError as exc: stats.blocked_sitemaps += 1 logger.warning("Sitemap index blocked, switching to direct probe: %s", exc) except SitemapDiscoveryError as exc: stats.malformed_sitemaps += 1 logger.warning("Sitemap index unusable, switching to direct probe: %s", exc) logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe") return _probe_direct_sitemap_urls(index_url, downloader, stats) def _probe_direct_sitemap_urls( index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats, ) -> list[str]: discovered: list[str] = [] consecutive_misses = 0 probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit) for sitemap_url in probe_urls: try: result = downloader.fetch(sitemap_url) raw_urls = list(_iter_loc_values(result.payload)) except SitemapBlockedError: stats.blocked_sitemaps += 1 consecutive_misses += 1 stats.direct_probe_misses += 1 if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses: break continue except SitemapDiscoveryError: consecutive_misses += 1 stats.direct_probe_misses += 1 if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses: break continue if not raw_urls: consecutive_misses += 1 stats.direct_probe_misses += 1 if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses: break continue consecutive_misses = 0 stats.direct_probe_hits += 1 discovered.append(sitemap_url) return discovered def _collect_vehicle_urls_from_sitemap( sitemap_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats, ) -> list[str]: logger.info("Downloading sitemap: %s", sitemap_url) try: result = downloader.fetch(sitemap_url) raw_urls = list(_iter_loc_values(result.payload)) except SitemapBlockedError as exc: stats.blocked_sitemaps += 1 logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc) return [] except SitemapDiscoveryError as exc: stats.malformed_sitemaps += 1 logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc) return [] stats.fetched_sitemaps += 1 vehicle_urls = _filter_vehicle_urls(raw_urls) if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls): logger.info("Skipping non-vehicle sitemap %s", sitemap_url) return [] logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls)) return vehicle_urls def discover_vehicle_urls_from_sitemap( index_url: str = DEFAULT_SITEMAP_INDEX_URL, *, settings: Settings | None = None, ) -> list[str]: result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings) return result.vehicle_urls def discover_vehicle_urls_from_sitemap_with_stats( index_url: str = DEFAULT_SITEMAP_INDEX_URL, *, settings: Settings | None = None, ) -> SitemapDiscoveryResult: runtime_settings = settings or Settings() downloader = _SitemapDownloader(runtime_settings) stats = SitemapDiscoveryStats(transport=downloader.transport_name) sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats) if not sitemap_urls: raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs") all_vehicle_urls: list[str] = [] for sitemap_url in sitemap_urls: all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats)) deduped = _filter_vehicle_urls(all_vehicle_urls) if not deduped: raise SitemapDiscoveryError( "No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue" ) logger.info( "Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)", len(deduped), stats.transport, stats.fetched_sitemaps, stats.blocked_sitemaps, stats.malformed_sitemaps, stats.direct_probe_hits, stats.direct_probe_misses, ) return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)