394 lines
17 KiB
Python
394 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import random
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from collections.abc import Callable, Iterable
|
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
|
|
|
import requests
|
|
|
|
from ..core.config import ProxyConfig
|
|
from .flight import extract_detail_listing, extract_search_results
|
|
from .models import MobileDeListing, MobileDeSearchPage
|
|
|
|
logger = logging.getLogger("mobile_de.client")
|
|
|
|
BASE_URL = "https://www.mobile.de"
|
|
SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
|
|
DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
|
|
DEFAULT_HEADERS = {
|
|
"user-agent": (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/124.0.0.0 Safari/537.36"
|
|
),
|
|
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"accept-language": "ru,en;q=0.9,de;q=0.8",
|
|
}
|
|
MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
|
|
MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
|
|
MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
|
|
MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
|
|
MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
|
|
MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
|
|
MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip()
|
|
MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120")))
|
|
MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS = max(1000, int(os.getenv("MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS", "60000")))
|
|
MOBILEDE_FLARESOLVERR_SESSION = os.getenv("MOBILEDE_FLARESOLVERR_SESSION", "").strip()
|
|
MOBILEDE_FLARESOLVERR_STATUSES = {
|
|
int(item.strip())
|
|
for item in os.getenv("MOBILEDE_FLARESOLVERR_STATUSES", "403,429,503").split(",")
|
|
if item.strip().isdigit()
|
|
}
|
|
|
|
|
|
class MobileDeClient:
|
|
"""HTTP client for mobile.de search/detail pages."""
|
|
|
|
def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None:
|
|
self.session = session or requests.Session()
|
|
self.session.headers.update(DEFAULT_HEADERS)
|
|
self.delay_seconds = max(0.0, delay_seconds)
|
|
self.proxy_config = ProxyConfig()
|
|
proxies = self.proxy_config.to_requests_proxies()
|
|
if proxies:
|
|
self.session.proxies.update(proxies)
|
|
|
|
@staticmethod
|
|
def _is_retryable_status(status_code: int) -> bool:
|
|
return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES
|
|
|
|
@staticmethod
|
|
def _should_use_flaresolverr(status_code: int | None) -> bool:
|
|
return (
|
|
MOBILEDE_FLARESOLVERR_ENABLED
|
|
and bool(MOBILEDE_FLARESOLVERR_URL)
|
|
and status_code is not None
|
|
and int(status_code) in MOBILEDE_FLARESOLVERR_STATUSES
|
|
)
|
|
|
|
@staticmethod
|
|
def _compute_backoff(attempt: int) -> float:
|
|
base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))
|
|
bounded = min(base, MOBILEDE_HTTP_BACKOFF_MAX_SECONDS) if MOBILEDE_HTTP_BACKOFF_MAX_SECONDS > 0 else base
|
|
jitter = random.uniform(0.0, MOBILEDE_HTTP_JITTER_SECONDS) if MOBILEDE_HTTP_JITTER_SECONDS > 0 else 0.0
|
|
return max(0.0, bounded + jitter)
|
|
|
|
@classmethod
|
|
def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient":
|
|
session = requests.Session()
|
|
adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
|
|
session.mount("https://", adapter)
|
|
session.mount("http://", adapter)
|
|
client = cls(session=session, delay_seconds=delay_seconds)
|
|
if client.proxy_config.enabled:
|
|
logger.info("mobile.de worker HTTP client using proxy: %s", client.proxy_config.server)
|
|
return client
|
|
|
|
@staticmethod
|
|
def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
|
|
make = str(make_id).strip()
|
|
model = str(model_id).strip() if model_id is not None else ""
|
|
return f"{make};{model};;"
|
|
|
|
@staticmethod
|
|
def build_search_url(page_number: int = 1, **params: str | int | None) -> str:
|
|
query = {
|
|
"sb": "rel",
|
|
"od": "up",
|
|
"vc": "Car",
|
|
"s": "Car",
|
|
"pageNumber": page_number,
|
|
}
|
|
query.update({key: value for key, value in params.items() if value is not None})
|
|
return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
|
|
|
|
@staticmethod
|
|
def build_search_url_from_existing(
|
|
search_url: str,
|
|
*,
|
|
page_number: int | None = None,
|
|
**params: str | int | None,
|
|
) -> str:
|
|
parts = urlsplit(search_url)
|
|
query_items = [
|
|
(key, value)
|
|
for key, value in parse_qsl(parts.query, keep_blank_values=True)
|
|
if key != "pageNumber" and key not in params
|
|
]
|
|
if page_number is not None:
|
|
query_items.append(("pageNumber", str(page_number)))
|
|
query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
|
|
scheme = parts.scheme or "https"
|
|
netloc = parts.netloc or urlsplit(BASE_URL).netloc
|
|
path = parts.path or SEARCH_PATH
|
|
return urlunsplit((scheme, netloc, path, urlencode(query_items), ""))
|
|
|
|
@staticmethod
|
|
def build_detail_url(listing_id: str | int) -> str:
|
|
query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"})
|
|
return f"{BASE_URL}{DETAIL_PATH}?{query}"
|
|
|
|
def fetch_html(self, url: str, *, timeout: int = 30) -> str:
|
|
last_error: Exception | None = None
|
|
attempts = max(1, MOBILEDE_HTTP_MAX_RETRIES + 1)
|
|
for attempt in range(1, attempts + 1):
|
|
try:
|
|
response = self.session.get(url, timeout=timeout)
|
|
if self._is_retryable_status(response.status_code):
|
|
if attempt < attempts:
|
|
sleep_seconds = self._compute_backoff(attempt)
|
|
logger.warning(
|
|
"mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
|
|
response.status_code,
|
|
attempt,
|
|
attempts,
|
|
sleep_seconds,
|
|
url,
|
|
)
|
|
if sleep_seconds:
|
|
time.sleep(sleep_seconds)
|
|
continue
|
|
if self._should_use_flaresolverr(response.status_code):
|
|
try:
|
|
return self._fetch_html_with_flaresolverr(url)
|
|
except Exception as exc:
|
|
logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, exc)
|
|
response.raise_for_status()
|
|
return response.text
|
|
except requests.RequestException as exc:
|
|
last_error = exc
|
|
status_code = getattr(getattr(exc, "response", None), "status_code", None)
|
|
retryable = bool(status_code is not None and self._is_retryable_status(int(status_code)))
|
|
if attempt >= attempts or not retryable:
|
|
if self._should_use_flaresolverr(status_code):
|
|
try:
|
|
return self._fetch_html_with_flaresolverr(url)
|
|
except Exception as flaresolverr_exc:
|
|
logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, flaresolverr_exc)
|
|
raise
|
|
sleep_seconds = self._compute_backoff(attempt)
|
|
logger.warning(
|
|
"mobile.de request error retry attempt=%s/%s status=%s sleep=%.2fs url=%s error=%s",
|
|
attempt,
|
|
attempts,
|
|
status_code,
|
|
sleep_seconds,
|
|
url,
|
|
exc,
|
|
)
|
|
if sleep_seconds:
|
|
time.sleep(sleep_seconds)
|
|
if last_error is not None:
|
|
raise last_error
|
|
raise RuntimeError("mobile.de fetch_html failed without a captured exception")
|
|
|
|
def _fetch_html_with_flaresolverr(self, url: str) -> str:
|
|
payload: dict[str, object] = {
|
|
"cmd": "request.get",
|
|
"url": url,
|
|
"maxTimeout": MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS,
|
|
}
|
|
if MOBILEDE_FLARESOLVERR_SESSION:
|
|
payload["session"] = MOBILEDE_FLARESOLVERR_SESSION
|
|
|
|
response = requests.post(
|
|
MOBILEDE_FLARESOLVERR_URL,
|
|
json=payload,
|
|
timeout=MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS,
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
if data.get("status") != "ok":
|
|
raise RuntimeError(str(data.get("message") or data))
|
|
|
|
solution = data.get("solution")
|
|
if not isinstance(solution, dict):
|
|
raise RuntimeError("FlareSolverr response does not contain solution")
|
|
|
|
html = solution.get("response")
|
|
if not isinstance(html, str) or not html:
|
|
raise RuntimeError("FlareSolverr response does not contain HTML")
|
|
|
|
user_agent = solution.get("userAgent")
|
|
if isinstance(user_agent, str) and user_agent:
|
|
self.session.headers.update({"user-agent": user_agent})
|
|
|
|
cookies = solution.get("cookies")
|
|
if isinstance(cookies, list):
|
|
for cookie in cookies:
|
|
if not isinstance(cookie, dict):
|
|
continue
|
|
name = cookie.get("name")
|
|
value = cookie.get("value")
|
|
if not isinstance(name, str) or not isinstance(value, str):
|
|
continue
|
|
self.session.cookies.set(
|
|
name,
|
|
value,
|
|
domain=cookie.get("domain") if isinstance(cookie.get("domain"), str) else None,
|
|
path=cookie.get("path") if isinstance(cookie.get("path"), str) else "/",
|
|
)
|
|
|
|
logger.info("mobile.de fetched via FlareSolverr url=%s", url)
|
|
return html
|
|
|
|
def fetch_search_page(
|
|
self,
|
|
page_number: int = 1,
|
|
*,
|
|
search_url: str | None = None,
|
|
**params: str | int | None,
|
|
) -> MobileDeSearchPage:
|
|
url = (
|
|
self.build_search_url_from_existing(search_url, page_number=page_number, **params)
|
|
if search_url
|
|
else self.build_search_url(page_number=page_number, **params)
|
|
)
|
|
html = self.fetch_html(url)
|
|
raw = extract_search_results(html)
|
|
listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
|
|
return MobileDeSearchPage(
|
|
url=url,
|
|
page_number=page_number,
|
|
total_results=raw.get("numResultsTotal"),
|
|
listings=listings,
|
|
raw_search_results=raw,
|
|
)
|
|
|
|
def iter_search_pages(
|
|
self,
|
|
*,
|
|
start_page: int = 1,
|
|
max_pages: int | None = None,
|
|
search_url: str | None = None,
|
|
stop_after_empty: bool = True,
|
|
progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
|
|
**params: str | int | None,
|
|
) -> Iterable[MobileDeSearchPage]:
|
|
page_number = start_page
|
|
pages_seen = 0
|
|
logger.debug(
|
|
"mobile.de search window started: start_page=%s max_pages=%s params=%s",
|
|
start_page,
|
|
max_pages,
|
|
{key: value for key, value in params.items() if value is not None},
|
|
)
|
|
while max_pages is None or pages_seen < max_pages:
|
|
logger.debug("mobile.de fetching search page=%s", page_number)
|
|
page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params)
|
|
page_meta = {
|
|
"page_number": page.page_number,
|
|
"pages_seen": pages_seen + 1,
|
|
"max_pages": max_pages,
|
|
"listing_count": len(page.listings),
|
|
"total_results": page.total_results,
|
|
}
|
|
logger.debug(
|
|
"mobile.de fetched search page=%s listings=%s total_results=%s",
|
|
page.page_number,
|
|
len(page.listings),
|
|
page.total_results,
|
|
)
|
|
if progress_callback is not None:
|
|
progress_callback(page, page_meta)
|
|
if stop_after_empty and not page.listings:
|
|
logger.debug("mobile.de stopping search window on empty page=%s", page.page_number)
|
|
break
|
|
yield page
|
|
pages_seen += 1
|
|
page_number += 1
|
|
if self.delay_seconds:
|
|
time.sleep(self.delay_seconds)
|
|
logger.debug(
|
|
"mobile.de search window finished: pages_seen=%s next_page=%s",
|
|
pages_seen,
|
|
page_number,
|
|
)
|
|
|
|
def fetch_search_pages_concurrent(
|
|
self,
|
|
*,
|
|
start_page: int = 1,
|
|
max_pages: int = 1,
|
|
workers: int = 8,
|
|
search_url: str | None = None,
|
|
stop_after_empty: bool = True,
|
|
progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
|
|
**params: str | int | None,
|
|
) -> list[MobileDeSearchPage]:
|
|
max_pages = max(1, int(max_pages))
|
|
workers = max(1, min(int(workers), max_pages))
|
|
page_numbers = list(range(max(1, int(start_page)), max(1, int(start_page)) + max_pages))
|
|
pages_by_number: dict[int, MobileDeSearchPage] = {}
|
|
thread_local = threading.local()
|
|
|
|
def _fetch_page(page_number: int) -> MobileDeSearchPage:
|
|
client = getattr(thread_local, "client", None)
|
|
if client is None:
|
|
client = MobileDeClient.for_worker(delay_seconds=0)
|
|
thread_local.client = client
|
|
return client.fetch_search_page(page_number=page_number, search_url=search_url, **params)
|
|
|
|
with ThreadPoolExecutor(max_workers=workers) as executor:
|
|
futures = {
|
|
executor.submit(_fetch_page, page_number): page_number
|
|
for page_number in page_numbers
|
|
}
|
|
for future in as_completed(futures):
|
|
page_number = futures[future]
|
|
page = future.result()
|
|
pages_by_number[page_number] = page
|
|
if progress_callback is not None:
|
|
progress_callback(
|
|
page,
|
|
{
|
|
"page_number": page.page_number,
|
|
"pages_seen": len(pages_by_number),
|
|
"max_pages": max_pages,
|
|
"listing_count": len(page.listings),
|
|
"total_results": page.total_results,
|
|
},
|
|
)
|
|
ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number]
|
|
if stop_after_empty:
|
|
non_empty_pages: list[MobileDeSearchPage] = []
|
|
for page in ordered_pages:
|
|
if not page.listings:
|
|
break
|
|
non_empty_pages.append(page)
|
|
return non_empty_pages
|
|
return ordered_pages
|
|
|
|
def fetch_detail(self, listing_id: str | int) -> dict:
|
|
html = self.fetch_html(self.build_detail_url(listing_id))
|
|
return extract_detail_listing(html)
|
|
|
|
@staticmethod
|
|
def _map_listing(item: dict) -> MobileDeListing:
|
|
listing_id = str(item.get("id") or item.get("adId") or "")
|
|
attr = item.get("attr") if isinstance(item.get("attr"), dict) else {}
|
|
contact = item.get("contact") if isinstance(item.get("contact"), dict) else {}
|
|
location = ", ".join(
|
|
part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part
|
|
) or None
|
|
return MobileDeListing(
|
|
id=listing_id,
|
|
url=MobileDeClient.build_detail_url(listing_id) if listing_id else "",
|
|
title=item.get("shortTitle"),
|
|
subtitle=item.get("subTitle"),
|
|
price=item.get("p"),
|
|
seller_name=contact.get("name"),
|
|
seller_type=contact.get("type") or item.get("st"),
|
|
location=location,
|
|
first_registration=attr.get("fr"),
|
|
mileage=attr.get("ml"),
|
|
power=attr.get("pw"),
|
|
fuel=attr.get("ft"),
|
|
transmission=attr.get("tr"),
|
|
raw=item,
|
|
)
|