qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix scraper flow

Browse Source
This commit is contained in:
qananasikq
2026-04-22 21:40:40 +03:00
parent 133c125e9c
commit d5d6ba8b7c
33 changed files with 2632 additions and 2811 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
iaai_scraper/discovery/__init__.py
View File

@@ -1,17 +1,15 @@
from .sitemap import (
DEFAULT_SITEMAP_INDEX_URL,
SitemapDiscoveryError,
SitemapBlockedError,
SitemapDiscoveryResult,
SitemapDiscoveryStats,
discover_vehicle_urls_from_sitemap,
discover_vehicle_urls_from_sitemap_with_stats,
)
__all__ = [
"DEFAULT_SITEMAP_INDEX_URL",
"SitemapBlockedError",
"SitemapDiscoveryError",
"SitemapDiscoveryResult",
"SitemapDiscoveryStats",
"discover_vehicle_urls_from_sitemap",
"discover_vehicle_urls_from_sitemap_with_stats",
]

455
iaai_scraper/discovery/sitemap.py
View File

@@ -3,222 +3,27 @@ from __future__ import annotations
import gzip
import io
import logging
import random
import re
import time
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Iterable
from urllib.error import HTTPError, URLError
from urllib.parse import urljoin, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from ..core.config import Settings
try:
from curl_cffi import requests as curl_requests
except ImportError: # pragma: no cover - optional runtime dependency guard
curl_requests = None
from urllib.parse import urlsplit, urlunsplit
from urllib.request import Request, urlopen, ProxyHandler, build_opener
import xml.etree.ElementTree as ET
logger = logging.getLogger("iaai_scraper.discovery.sitemap")
DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
_SITEMAP_TIMEOUT_SECONDS = 30
_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
_XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL)
_BLOCK_MARKERS = (
b"pardon our interruption",
b"incapsula",
b"please stand by",
b"_incapsula_resource",
b"as you were browsing something about your browser",
)
_HTML_MARKERS = (b"<html", b"<!doctype html", b"<script", b"document.getelementsbyclassname")
_CURL_IMPERSONATE_CHOICES = ("chrome136", "chrome133", "chrome131", "safari184")
class SitemapDiscoveryError(RuntimeError):
pass
class SitemapBlockedError(SitemapDiscoveryError):
pass
@dataclass(slots=True)
class SitemapFetchResult:
url: str
payload: bytes
source: str
blocked: bool = False
status_code: int | None = None
content_type: str | None = None
@dataclass(slots=True)
class SitemapDiscoveryStats:
transport: str
fetched_sitemaps: int = 0
blocked_sitemaps: int = 0
malformed_sitemaps: int = 0
direct_probe_hits: int = 0
direct_probe_misses: int = 0
@dataclass(slots=True)
class SitemapDiscoveryResult:
vehicle_urls: list[str]
stats: SitemapDiscoveryStats
_DEFAULT_HEADERS = {
"Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Pragma": "no-cache",
"Upgrade-Insecure-Requests": "1",
}
class _SitemapDownloader:
def __init__(self, settings: Settings) -> None:
self.settings = settings
self.discovery = settings.discovery
self.proxy_url = settings.proxy.server
self.proxy_auth = None
if settings.proxy.username:
password = settings.proxy.password or ""
self.proxy_auth = (settings.proxy.username, password)
self._transport_name = self._resolve_transport_name()
@property
def transport_name(self) -> str:
return self._transport_name
def _resolve_transport_name(self) -> str:
if self.discovery.sitemap_use_curl_cffi and curl_requests is not None:
return "curl_cffi"
return "urllib"
def fetch(self, url: str) -> SitemapFetchResult:
attempts = max(1, int(self.discovery.sitemap_retry_attempts))
last_exc: Exception | None = None
for attempt in range(1, attempts + 1):
try:
result = self._fetch_once(url)
if result.blocked:
raise SitemapBlockedError(f"Anti-bot page returned for {url}")
return result
except SitemapBlockedError as exc:
last_exc = exc
logger.warning(
"Sitemap fetch blocked (attempt %d/%d, transport=%s): %s",
attempt,
attempts,
self.transport_name,
url,
)
except Exception as exc:
last_exc = exc
logger.warning(
"Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s",
attempt,
attempts,
self.transport_name,
url,
exc,
)
if attempt >= attempts:
break
time.sleep(self._backoff_delay(attempt))
if last_exc is None:
raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}")
if isinstance(last_exc, SitemapDiscoveryError):
raise last_exc
raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc
def _fetch_once(self, url: str) -> SitemapFetchResult:
if self.transport_name == "curl_cffi":
return self._fetch_with_curl_cffi(url)
return self._fetch_with_urllib(url)
def _build_headers(self) -> dict[str, str]:
headers = dict(_DEFAULT_HEADERS)
headers["User-Agent"] = self.settings.fingerprint.user_agent
return headers
def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult:
assert curl_requests is not None
response = curl_requests.get(
url,
headers=self._build_headers(),
timeout=self.discovery.sitemap_timeout_seconds,
impersonate=random.choice(_CURL_IMPERSONATE_CHOICES),
proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None,
proxy_auth=self.proxy_auth,
allow_redirects=True,
)
payload = self._decode_payload(
payload=response.content,
encoding=(response.headers.get("Content-Encoding") or ""),
url=url,
)
blocked = _looks_like_block_page(payload)
return SitemapFetchResult(
url=url,
payload=payload,
source="curl_cffi",
blocked=blocked,
status_code=int(response.status_code),
content_type=response.headers.get("Content-Type"),
)
def _fetch_with_urllib(self, url: str) -> SitemapFetchResult:
request = Request(url, headers=self._build_headers())
try:
with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response:
payload = response.read()
decoded = self._decode_payload(
payload=payload,
encoding=(response.headers.get("Content-Encoding") or ""),
url=url,
)
return SitemapFetchResult(
url=url,
payload=decoded,
source="urllib",
blocked=_looks_like_block_page(decoded),
status_code=getattr(response, "status", None),
content_type=response.headers.get("Content-Type"),
)
except HTTPError as exc:
payload = exc.read() if hasattr(exc, "read") else b""
decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url)
blocked = exc.code in {403, 429} or _looks_like_block_page(decoded)
if blocked:
raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc
raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc
except URLError as exc:
raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc
@staticmethod
def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes:
normalized = (encoding or "").lower().strip()
if normalized == "gzip" or url.lower().endswith(".gz"):
try:
return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
except OSError:
return payload
return payload
def _backoff_delay(self, attempt: int) -> float:
base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds))
jitter = random.uniform(0.05, 0.35)
return base * (2 ** (attempt - 1)) + jitter
def _is_vehicle_sitemap_url(url: str) -> bool:
lowered = url.strip().lower()
# Берём только sitemap с авто.
if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
return False
return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
@@ -229,21 +34,38 @@ def _normalize_vehicle_url(url: str) -> str:
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
request = Request(
url,
headers={
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
),
"Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
"Accept-Encoding": "gzip",
},
)
if proxy_url:
handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
opener = build_opener(handler)
response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
else:
response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
with response:
payload = response.read()
encoding = str(response.headers.get("Content-Encoding") or "").lower()
if encoding == "gzip" or url.lower().endswith(".gz"):
return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
return payload
def _local_name(tag: str) -> str:
if "}" in tag:
return tag.rsplit("}", 1)[1]
return tag
def _looks_like_block_page(payload: bytes) -> bool:
sample = payload[:8192].lower()
if any(marker in sample for marker in _BLOCK_MARKERS):
return True
if any(marker in sample for marker in _HTML_MARKERS) and b"<loc>" not in sample:
return True
return False
def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
for match in _LOC_TAG_RE.finditer(xml_bytes):
try:
@@ -255,13 +77,6 @@ def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
if _looks_like_block_page(xml_bytes):
raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML")
if not _XML_PREFIX_RE.search(xml_bytes[:256]):
fallback_values = list(_iter_loc_values_fallback(xml_bytes))
if fallback_values:
yield from fallback_values
return
try:
root = ET.fromstring(xml_bytes)
except ET.ParseError as exc:
@@ -303,139 +118,93 @@ def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
return any("/vehicledetail/" in url.lower() for url in urls)
def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]:
base_dir = index_url.rsplit("/", 1)[0] + "/"
return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)]
def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]:
def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
logger.info("Downloading sitemap index: %s", index_url)
try:
index_result = downloader.fetch(index_url)
sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)]
if sitemap_urls:
return sitemap_urls
except SitemapBlockedError as exc:
stats.blocked_sitemaps += 1
logger.warning("Sitemap index blocked, switching to direct probe: %s", exc)
except SitemapDiscoveryError as exc:
stats.malformed_sitemaps += 1
logger.warning("Sitemap index unusable, switching to direct probe: %s", exc)
logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe")
return _probe_direct_sitemap_urls(index_url, downloader, stats)
def _probe_direct_sitemap_urls(
index_url: str,
downloader: _SitemapDownloader,
stats: SitemapDiscoveryStats,
) -> list[str]:
discovered: list[str] = []
consecutive_misses = 0
probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit)
for sitemap_url in probe_urls:
try:
result = downloader.fetch(sitemap_url)
raw_urls = list(_iter_loc_values(result.payload))
except SitemapBlockedError:
stats.blocked_sitemaps += 1
consecutive_misses += 1
stats.direct_probe_misses += 1
if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
break
continue
except SitemapDiscoveryError:
consecutive_misses += 1
stats.direct_probe_misses += 1
if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
break
continue
if not raw_urls:
consecutive_misses += 1
stats.direct_probe_misses += 1
if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
break
continue
consecutive_misses = 0
stats.direct_probe_hits += 1
discovered.append(sitemap_url)
return discovered
def _collect_vehicle_urls_from_sitemap(
sitemap_url: str,
downloader: _SitemapDownloader,
stats: SitemapDiscoveryStats,
) -> list[str]:
logger.info("Downloading sitemap: %s", sitemap_url)
try:
result = downloader.fetch(sitemap_url)
raw_urls = list(_iter_loc_values(result.payload))
except SitemapBlockedError as exc:
stats.blocked_sitemaps += 1
logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc)
return []
except SitemapDiscoveryError as exc:
stats.malformed_sitemaps += 1
logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
return []
stats.fetched_sitemaps += 1
vehicle_urls = _filter_vehicle_urls(raw_urls)
if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
return []
logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
return vehicle_urls
def discover_vehicle_urls_from_sitemap(
index_url: str = DEFAULT_SITEMAP_INDEX_URL,
*,
settings: Settings | None = None,
) -> list[str]:
result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings)
return result.vehicle_urls
def discover_vehicle_urls_from_sitemap_with_stats(
index_url: str = DEFAULT_SITEMAP_INDEX_URL,
*,
settings: Settings | None = None,
) -> SitemapDiscoveryResult:
runtime_settings = settings or Settings()
downloader = _SitemapDownloader(runtime_settings)
stats = SitemapDiscoveryStats(transport=downloader.transport_name)
sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats)
index_xml = _download_bytes(index_url, proxy_url=proxy_url)
sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
if not sitemap_urls:
raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs")
raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
all_vehicle_urls: list[str] = []
for sitemap_url in sitemap_urls:
all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats))
logger.info("Downloading sitemap: %s", sitemap_url)
try:
sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
raw_urls = list(_iter_loc_values(sitemap_xml))
except SitemapDiscoveryError as exc:
logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
continue
vehicle_urls = _filter_vehicle_urls(raw_urls)
if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
continue
logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
all_vehicle_urls.extend(vehicle_urls)
deduped = _filter_vehicle_urls(all_vehicle_urls)
if not deduped:
raise SitemapDiscoveryError(
"No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue"
)
raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
return deduped
logger.info(
"Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)",
len(deduped),
stats.transport,
stats.fetched_sitemaps,
stats.blocked_sitemaps,
stats.malformed_sitemaps,
stats.direct_probe_hits,
stats.direct_probe_misses,
)
@dataclass
class SitemapDiscoveryStats:
transport: str = "http"
fetched_sitemaps: int = 0
blocked_sitemaps: int = 0
malformed_sitemaps: int = 0
direct_probe_hits: int = 0
direct_probe_misses: int = 0
@dataclass
class SitemapDiscoveryResult:
vehicle_urls: list[str] = field(default_factory=list)
stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
def discover_vehicle_urls_from_sitemap_with_stats(
settings=None,
index_url: str = DEFAULT_SITEMAP_INDEX_URL,
) -> SitemapDiscoveryResult:
"""Возвращает URL и статистику."""
proxy_url = None
if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
proxy_url = settings.proxy.server
stats = SitemapDiscoveryStats(transport="http")
logger.info("Downloading sitemap index: %s", index_url)
index_xml = _download_bytes(index_url, proxy_url=proxy_url)
sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
if not sitemap_urls:
raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
all_vehicle_urls: list[str] = []
for sitemap_url in sitemap_urls:
logger.info("Downloading sitemap: %s", sitemap_url)
try:
sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
raw_urls = list(_iter_loc_values(sitemap_xml))
stats.fetched_sitemaps += 1
except SitemapDiscoveryError as exc:
logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
stats.malformed_sitemaps += 1
continue
except Exception as exc:
logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
stats.blocked_sitemaps += 1
continue
vehicle_urls = _filter_vehicle_urls(raw_urls)
if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
continue
logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
all_vehicle_urls.extend(vehicle_urls)
deduped = _filter_vehicle_urls(all_vehicle_urls)
if not deduped:
raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)