442 lines
15 KiB
Python
442 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import gzip
|
|
import io
|
|
import logging
|
|
import random
|
|
import re
|
|
import time
|
|
import xml.etree.ElementTree as ET
|
|
from dataclasses import dataclass
|
|
from typing import Iterable
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urljoin, urlsplit, urlunsplit
|
|
from urllib.request import Request, urlopen
|
|
|
|
from ..core.config import Settings
|
|
|
|
try:
|
|
from curl_cffi import requests as curl_requests
|
|
except ImportError: # pragma: no cover - optional runtime dependency guard
|
|
curl_requests = None
|
|
|
|
logger = logging.getLogger("iaai_scraper.discovery.sitemap")
|
|
|
|
DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
|
|
_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
|
|
_XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL)
|
|
_BLOCK_MARKERS = (
|
|
b"pardon our interruption",
|
|
b"incapsula",
|
|
b"please stand by",
|
|
b"_incapsula_resource",
|
|
b"as you were browsing something about your browser",
|
|
)
|
|
_HTML_MARKERS = (b"<html", b"<!doctype html", b"<script", b"document.getelementsbyclassname")
|
|
_CURL_IMPERSONATE_CHOICES = ("chrome136", "chrome133", "chrome131", "safari184")
|
|
|
|
|
|
class SitemapDiscoveryError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class SitemapBlockedError(SitemapDiscoveryError):
|
|
pass
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class SitemapFetchResult:
|
|
url: str
|
|
payload: bytes
|
|
source: str
|
|
blocked: bool = False
|
|
status_code: int | None = None
|
|
content_type: str | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class SitemapDiscoveryStats:
|
|
transport: str
|
|
fetched_sitemaps: int = 0
|
|
blocked_sitemaps: int = 0
|
|
malformed_sitemaps: int = 0
|
|
direct_probe_hits: int = 0
|
|
direct_probe_misses: int = 0
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class SitemapDiscoveryResult:
|
|
vehicle_urls: list[str]
|
|
stats: SitemapDiscoveryStats
|
|
|
|
|
|
_DEFAULT_HEADERS = {
|
|
"Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
|
|
"Accept-Encoding": "gzip, deflate, br",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
"Cache-Control": "no-cache",
|
|
"Pragma": "no-cache",
|
|
"Upgrade-Insecure-Requests": "1",
|
|
}
|
|
|
|
|
|
class _SitemapDownloader:
|
|
def __init__(self, settings: Settings) -> None:
|
|
self.settings = settings
|
|
self.discovery = settings.discovery
|
|
self.proxy_url = settings.proxy.server
|
|
self.proxy_auth = None
|
|
if settings.proxy.username:
|
|
password = settings.proxy.password or ""
|
|
self.proxy_auth = (settings.proxy.username, password)
|
|
self._transport_name = self._resolve_transport_name()
|
|
|
|
@property
|
|
def transport_name(self) -> str:
|
|
return self._transport_name
|
|
|
|
def _resolve_transport_name(self) -> str:
|
|
if self.discovery.sitemap_use_curl_cffi and curl_requests is not None:
|
|
return "curl_cffi"
|
|
return "urllib"
|
|
|
|
def fetch(self, url: str) -> SitemapFetchResult:
|
|
attempts = max(1, int(self.discovery.sitemap_retry_attempts))
|
|
last_exc: Exception | None = None
|
|
for attempt in range(1, attempts + 1):
|
|
try:
|
|
result = self._fetch_once(url)
|
|
if result.blocked:
|
|
raise SitemapBlockedError(f"Anti-bot page returned for {url}")
|
|
return result
|
|
except SitemapBlockedError as exc:
|
|
last_exc = exc
|
|
logger.warning(
|
|
"Sitemap fetch blocked (attempt %d/%d, transport=%s): %s",
|
|
attempt,
|
|
attempts,
|
|
self.transport_name,
|
|
url,
|
|
)
|
|
except Exception as exc:
|
|
last_exc = exc
|
|
logger.warning(
|
|
"Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s",
|
|
attempt,
|
|
attempts,
|
|
self.transport_name,
|
|
url,
|
|
exc,
|
|
)
|
|
if attempt >= attempts:
|
|
break
|
|
time.sleep(self._backoff_delay(attempt))
|
|
if last_exc is None:
|
|
raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}")
|
|
if isinstance(last_exc, SitemapDiscoveryError):
|
|
raise last_exc
|
|
raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc
|
|
|
|
def _fetch_once(self, url: str) -> SitemapFetchResult:
|
|
if self.transport_name == "curl_cffi":
|
|
return self._fetch_with_curl_cffi(url)
|
|
return self._fetch_with_urllib(url)
|
|
|
|
def _build_headers(self) -> dict[str, str]:
|
|
headers = dict(_DEFAULT_HEADERS)
|
|
headers["User-Agent"] = self.settings.fingerprint.user_agent
|
|
return headers
|
|
|
|
def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult:
|
|
assert curl_requests is not None
|
|
response = curl_requests.get(
|
|
url,
|
|
headers=self._build_headers(),
|
|
timeout=self.discovery.sitemap_timeout_seconds,
|
|
impersonate=random.choice(_CURL_IMPERSONATE_CHOICES),
|
|
proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None,
|
|
proxy_auth=self.proxy_auth,
|
|
allow_redirects=True,
|
|
)
|
|
payload = self._decode_payload(
|
|
payload=response.content,
|
|
encoding=(response.headers.get("Content-Encoding") or ""),
|
|
url=url,
|
|
)
|
|
blocked = _looks_like_block_page(payload)
|
|
return SitemapFetchResult(
|
|
url=url,
|
|
payload=payload,
|
|
source="curl_cffi",
|
|
blocked=blocked,
|
|
status_code=int(response.status_code),
|
|
content_type=response.headers.get("Content-Type"),
|
|
)
|
|
|
|
def _fetch_with_urllib(self, url: str) -> SitemapFetchResult:
|
|
request = Request(url, headers=self._build_headers())
|
|
try:
|
|
with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response:
|
|
payload = response.read()
|
|
decoded = self._decode_payload(
|
|
payload=payload,
|
|
encoding=(response.headers.get("Content-Encoding") or ""),
|
|
url=url,
|
|
)
|
|
return SitemapFetchResult(
|
|
url=url,
|
|
payload=decoded,
|
|
source="urllib",
|
|
blocked=_looks_like_block_page(decoded),
|
|
status_code=getattr(response, "status", None),
|
|
content_type=response.headers.get("Content-Type"),
|
|
)
|
|
except HTTPError as exc:
|
|
payload = exc.read() if hasattr(exc, "read") else b""
|
|
decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url)
|
|
blocked = exc.code in {403, 429} or _looks_like_block_page(decoded)
|
|
if blocked:
|
|
raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc
|
|
raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc
|
|
except URLError as exc:
|
|
raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc
|
|
|
|
@staticmethod
|
|
def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes:
|
|
normalized = (encoding or "").lower().strip()
|
|
if normalized == "gzip" or url.lower().endswith(".gz"):
|
|
try:
|
|
return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
|
|
except OSError:
|
|
return payload
|
|
return payload
|
|
|
|
def _backoff_delay(self, attempt: int) -> float:
|
|
base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds))
|
|
jitter = random.uniform(0.05, 0.35)
|
|
return base * (2 ** (attempt - 1)) + jitter
|
|
|
|
|
|
def _is_vehicle_sitemap_url(url: str) -> bool:
|
|
lowered = url.strip().lower()
|
|
if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
|
|
return False
|
|
return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
|
|
|
|
|
|
def _normalize_vehicle_url(url: str) -> str:
|
|
parts = urlsplit(url.strip())
|
|
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
|
|
|
|
|
def _local_name(tag: str) -> str:
|
|
if "}" in tag:
|
|
return tag.rsplit("}", 1)[1]
|
|
return tag
|
|
|
|
|
|
def _looks_like_block_page(payload: bytes) -> bool:
|
|
sample = payload[:8192].lower()
|
|
if any(marker in sample for marker in _BLOCK_MARKERS):
|
|
return True
|
|
if any(marker in sample for marker in _HTML_MARKERS) and b"<loc>" not in sample:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
|
|
for match in _LOC_TAG_RE.finditer(xml_bytes):
|
|
try:
|
|
value = match.group(1).decode("utf-8", errors="ignore").strip()
|
|
except Exception:
|
|
continue
|
|
if value:
|
|
yield value
|
|
|
|
|
|
def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
|
|
if _looks_like_block_page(xml_bytes):
|
|
raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML")
|
|
if not _XML_PREFIX_RE.search(xml_bytes[:256]):
|
|
fallback_values = list(_iter_loc_values_fallback(xml_bytes))
|
|
if fallback_values:
|
|
yield from fallback_values
|
|
return
|
|
try:
|
|
root = ET.fromstring(xml_bytes)
|
|
except ET.ParseError as exc:
|
|
fallback_values = list(_iter_loc_values_fallback(xml_bytes))
|
|
if fallback_values:
|
|
logger.warning(
|
|
"Falling back to regex sitemap loc extraction after XML parse error: %s",
|
|
exc,
|
|
)
|
|
yield from fallback_values
|
|
return
|
|
raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
|
|
|
|
for element in root.iter():
|
|
if _local_name(element.tag) != "loc":
|
|
continue
|
|
if not element.text:
|
|
continue
|
|
value = element.text.strip()
|
|
if value:
|
|
yield value
|
|
|
|
|
|
def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
|
|
result: list[str] = []
|
|
seen: set[str] = set()
|
|
for url in urls:
|
|
normalized = _normalize_vehicle_url(url)
|
|
if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
|
|
continue
|
|
if normalized in seen:
|
|
continue
|
|
seen.add(normalized)
|
|
result.append(normalized)
|
|
return result
|
|
|
|
|
|
def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
|
|
return any("/vehicledetail/" in url.lower() for url in urls)
|
|
|
|
|
|
def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]:
|
|
base_dir = index_url.rsplit("/", 1)[0] + "/"
|
|
return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)]
|
|
|
|
|
|
def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]:
|
|
logger.info("Downloading sitemap index: %s", index_url)
|
|
try:
|
|
index_result = downloader.fetch(index_url)
|
|
sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)]
|
|
if sitemap_urls:
|
|
return sitemap_urls
|
|
except SitemapBlockedError as exc:
|
|
stats.blocked_sitemaps += 1
|
|
logger.warning("Sitemap index blocked, switching to direct probe: %s", exc)
|
|
except SitemapDiscoveryError as exc:
|
|
stats.malformed_sitemaps += 1
|
|
logger.warning("Sitemap index unusable, switching to direct probe: %s", exc)
|
|
|
|
logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe")
|
|
return _probe_direct_sitemap_urls(index_url, downloader, stats)
|
|
|
|
|
|
def _probe_direct_sitemap_urls(
|
|
index_url: str,
|
|
downloader: _SitemapDownloader,
|
|
stats: SitemapDiscoveryStats,
|
|
) -> list[str]:
|
|
discovered: list[str] = []
|
|
consecutive_misses = 0
|
|
probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit)
|
|
|
|
for sitemap_url in probe_urls:
|
|
try:
|
|
result = downloader.fetch(sitemap_url)
|
|
raw_urls = list(_iter_loc_values(result.payload))
|
|
except SitemapBlockedError:
|
|
stats.blocked_sitemaps += 1
|
|
consecutive_misses += 1
|
|
stats.direct_probe_misses += 1
|
|
if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
|
|
break
|
|
continue
|
|
except SitemapDiscoveryError:
|
|
consecutive_misses += 1
|
|
stats.direct_probe_misses += 1
|
|
if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
|
|
break
|
|
continue
|
|
|
|
if not raw_urls:
|
|
consecutive_misses += 1
|
|
stats.direct_probe_misses += 1
|
|
if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
|
|
break
|
|
continue
|
|
|
|
consecutive_misses = 0
|
|
stats.direct_probe_hits += 1
|
|
discovered.append(sitemap_url)
|
|
|
|
return discovered
|
|
|
|
|
|
def _collect_vehicle_urls_from_sitemap(
|
|
sitemap_url: str,
|
|
downloader: _SitemapDownloader,
|
|
stats: SitemapDiscoveryStats,
|
|
) -> list[str]:
|
|
logger.info("Downloading sitemap: %s", sitemap_url)
|
|
try:
|
|
result = downloader.fetch(sitemap_url)
|
|
raw_urls = list(_iter_loc_values(result.payload))
|
|
except SitemapBlockedError as exc:
|
|
stats.blocked_sitemaps += 1
|
|
logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc)
|
|
return []
|
|
except SitemapDiscoveryError as exc:
|
|
stats.malformed_sitemaps += 1
|
|
logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
|
|
return []
|
|
|
|
stats.fetched_sitemaps += 1
|
|
vehicle_urls = _filter_vehicle_urls(raw_urls)
|
|
if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
|
|
logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
|
|
return []
|
|
|
|
logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
|
|
return vehicle_urls
|
|
|
|
|
|
def discover_vehicle_urls_from_sitemap(
|
|
index_url: str = DEFAULT_SITEMAP_INDEX_URL,
|
|
*,
|
|
settings: Settings | None = None,
|
|
) -> list[str]:
|
|
result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings)
|
|
return result.vehicle_urls
|
|
|
|
|
|
|
|
def discover_vehicle_urls_from_sitemap_with_stats(
|
|
index_url: str = DEFAULT_SITEMAP_INDEX_URL,
|
|
*,
|
|
settings: Settings | None = None,
|
|
) -> SitemapDiscoveryResult:
|
|
runtime_settings = settings or Settings()
|
|
downloader = _SitemapDownloader(runtime_settings)
|
|
stats = SitemapDiscoveryStats(transport=downloader.transport_name)
|
|
|
|
sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats)
|
|
if not sitemap_urls:
|
|
raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs")
|
|
|
|
all_vehicle_urls: list[str] = []
|
|
for sitemap_url in sitemap_urls:
|
|
all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats))
|
|
|
|
deduped = _filter_vehicle_urls(all_vehicle_urls)
|
|
if not deduped:
|
|
raise SitemapDiscoveryError(
|
|
"No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue"
|
|
)
|
|
|
|
logger.info(
|
|
"Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)",
|
|
len(deduped),
|
|
stats.transport,
|
|
stats.fetched_sitemaps,
|
|
stats.blocked_sitemaps,
|
|
stats.malformed_sitemaps,
|
|
stats.direct_probe_hits,
|
|
stats.direct_probe_misses,
|
|
)
|
|
return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)
|