qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix parser identifiers and locale

Browse Source
This commit is contained in:
qananasikq
2026-08-05 18:13:30 +03:00
parent 2cebb78316
commit fba7299747
5 changed files with 185 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
mobilede_scraper/mobile_de/client.py
View File

@@ -17,9 +17,9 @@ from .models import MobileDeListing, MobileDeSearchPage
logger = logging.getLogger("mobile_de.client")
BASE_URL = "https://www.mobile.de"
SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
BASE_URL = "https://suchen.mobile.de"
SEARCH_PATH = "/fahrzeuge/search.html"
DETAIL_PATH = "/fahrzeuge/details.html"
DEFAULT_HEADERS = {
"user-agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -27,7 +27,7 @@ DEFAULT_HEADERS = {
"Chrome/124.0.0.0 Safari/537.36"
),
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"accept-language": "ru,en;q=0.9,de;q=0.8",
"accept-language": "en-US,en;q=0.9",
}
MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4")))
MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
@@ -103,6 +103,7 @@ class MobileDeClient:
"od": "up",
"vc": "Car",
"s": "Car",
"lang": "en",
"pageNumber": page_number,
}
query.update({key: value for key, value in params.items() if value is not None})
@@ -119,19 +120,17 @@ class MobileDeClient:
query_items = [
(key, value)
for key, value in parse_qsl(parts.query, keep_blank_values=True)
if key != "pageNumber" and key not in params
if key != "pageNumber" and key != "lang" and key not in params
]
if page_number is not None:
query_items.append(("pageNumber", str(page_number)))
query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
scheme = parts.scheme or "https"
netloc = parts.netloc or urlsplit(BASE_URL).netloc
path = parts.path or SEARCH_PATH
return urlunsplit((scheme, netloc, path, urlencode(query_items), ""))
query_items.append(("lang", "en"))
return urlunsplit(("https", urlsplit(BASE_URL).netloc, SEARCH_PATH, urlencode(query_items), ""))
@staticmethod
def build_detail_url(listing_id: str | int) -> str:
query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"})
query = urlencode({"id": listing_id, "vc": "Car", "s": "Car", "lang": "en"})
return f"{BASE_URL}{DETAIL_PATH}?{query}"
def fetch_html(self, url: str, *, timeout: int = 30) -> str: