qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor mobile.de parser, fix country mapping, update README

Browse Source
This commit is contained in:
qananasikq
2026-05-04 18:12:10 +03:00
parent edddfc884c
commit 800867b4e6
46 changed files with 2887 additions and 10900 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
mobilede_scraper/mobile_de/client.py
View File

@@ -3,7 +3,6 @@ from __future__ import annotations
import logging
import os
import random
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from collections.abc import Callable, Iterable
@@ -34,6 +33,16 @@ MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BAC
MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip()
MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120")))
MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS = max(1000, int(os.getenv("MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS", "60000")))
MOBILEDE_FLARESOLVERR_SESSION = os.getenv("MOBILEDE_FLARESOLVERR_SESSION", "").strip()
MOBILEDE_FLARESOLVERR_STATUSES = {
int(item.strip())
for item in os.getenv("MOBILEDE_FLARESOLVERR_STATUSES", "403,429,503").split(",")
if item.strip().isdigit()
}
class MobileDeClient:
@@ -43,11 +52,24 @@ class MobileDeClient:
self.session = session or requests.Session()
self.session.headers.update(DEFAULT_HEADERS)
self.delay_seconds = max(0.0, delay_seconds)
self.proxy_config = ProxyConfig()
proxies = self.proxy_config.to_requests_proxies()
if proxies:
self.session.proxies.update(proxies)
@staticmethod
def _is_retryable_status(status_code: int) -> bool:
return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES
@staticmethod
def _should_use_flaresolverr(status_code: int | None) -> bool:
return (
MOBILEDE_FLARESOLVERR_ENABLED
and bool(MOBILEDE_FLARESOLVERR_URL)
and status_code is not None
and int(status_code) in MOBILEDE_FLARESOLVERR_STATUSES
)
@staticmethod
def _compute_backoff(attempt: int) -> float:
base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))
@@ -61,12 +83,10 @@ class MobileDeClient:
adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
session.mount("https://", adapter)
session.mount("http://", adapter)
proxy_cfg = ProxyConfig()
proxies = proxy_cfg.to_requests_proxies()
if proxies:
session.proxies.update(proxies)
logger.info("mobile.de worker HTTP client using proxy: %s", proxy_cfg.server)
return cls(session=session, delay_seconds=delay_seconds)
client = cls(session=session, delay_seconds=delay_seconds)
if client.proxy_config.enabled:
logger.info("mobile.de worker HTTP client using proxy: %s", client.proxy_config.server)
return client
@staticmethod
def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
@@ -118,19 +138,25 @@ class MobileDeClient:
for attempt in range(1, attempts + 1):
try:
response = self.session.get(url, timeout=timeout)
if self._is_retryable_status(response.status_code) and attempt < attempts:
sleep_seconds = self._compute_backoff(attempt)
logger.warning(
"mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
response.status_code,
attempt,
attempts,
sleep_seconds,
url,
)
if sleep_seconds:
time.sleep(sleep_seconds)
continue
if self._is_retryable_status(response.status_code):
if attempt < attempts:
sleep_seconds = self._compute_backoff(attempt)
logger.warning(
"mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
response.status_code,
attempt,
attempts,
sleep_seconds,
url,
)
if sleep_seconds:
time.sleep(sleep_seconds)
continue
if self._should_use_flaresolverr(response.status_code):
try:
return self._fetch_html_with_flaresolverr(url)
except Exception as exc:
logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, exc)
response.raise_for_status()
return response.text
except requests.RequestException as exc:
@@ -138,6 +164,11 @@ class MobileDeClient:
status_code = getattr(getattr(exc, "response", None), "status_code", None)
retryable = bool(status_code is not None and self._is_retryable_status(int(status_code)))
if attempt >= attempts or not retryable:
if self._should_use_flaresolverr(status_code):
try:
return self._fetch_html_with_flaresolverr(url)
except Exception as flaresolverr_exc:
logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, flaresolverr_exc)
raise
sleep_seconds = self._compute_backoff(attempt)
logger.warning(
@@ -155,6 +186,56 @@ class MobileDeClient:
raise last_error
raise RuntimeError("mobile.de fetch_html failed without a captured exception")
def _fetch_html_with_flaresolverr(self, url: str) -> str:
payload: dict[str, object] = {
"cmd": "request.get",
"url": url,
"maxTimeout": MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS,
}
if MOBILEDE_FLARESOLVERR_SESSION:
payload["session"] = MOBILEDE_FLARESOLVERR_SESSION
response = requests.post(
MOBILEDE_FLARESOLVERR_URL,
json=payload,
timeout=MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS,
)
response.raise_for_status()
data = response.json()
if data.get("status") != "ok":
raise RuntimeError(str(data.get("message") or data))
solution = data.get("solution")
if not isinstance(solution, dict):
raise RuntimeError("FlareSolverr response does not contain solution")
html = solution.get("response")
if not isinstance(html, str) or not html:
raise RuntimeError("FlareSolverr response does not contain HTML")
user_agent = solution.get("userAgent")
if isinstance(user_agent, str) and user_agent:
self.session.headers.update({"user-agent": user_agent})
cookies = solution.get("cookies")
if isinstance(cookies, list):
for cookie in cookies:
if not isinstance(cookie, dict):
continue
name = cookie.get("name")
value = cookie.get("value")
if not isinstance(name, str) or not isinstance(value, str):
continue
self.session.cookies.set(
name,
value,
domain=cookie.get("domain") if isinstance(cookie.get("domain"), str) else None,
path=cookie.get("path") if isinstance(cookie.get("path"), str) else "/",
)
logger.info("mobile.de fetched via FlareSolverr url=%s", url)
return html
def fetch_search_page(
self,
page_number: int = 1,