qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve runtime sync flow

Browse Source
This commit is contained in:
qananasikq
2026-05-11 12:45:51 +03:00
parent 124dfc514f
commit d9d073641b
13 changed files with 1224 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
mobilede_scraper/mobile_de/client.py
View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import logging
import os
import random
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from collections.abc import Callable, Iterable
@@ -33,6 +34,7 @@ MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BAC
MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
MOBILEDE_DETAIL_TIMEOUT_SECONDS = max(1, int(float(os.getenv("MOBILEDE_DETAIL_TIMEOUT_SECONDS", "15"))))
MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip()
MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120")))
@@ -241,6 +243,8 @@ class MobileDeClient:
page_number: int = 1,
*,
search_url: str | None = None,
timeout: int = 30,
max_retries: int | None = None,
**params: str | int | None,
) -> MobileDeSearchPage:
url = (
@@ -248,7 +252,15 @@ class MobileDeClient:
if search_url
else self.build_search_url(page_number=page_number, **params)
)
html = self.fetch_html(url)
if max_retries is None:
html = self.fetch_html(url, timeout=timeout)
else:
previous_retries = os.environ.get("MOBILEDE_HTTP_MAX_RETRIES")
try:
globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(max_retries))
html = self.fetch_html(url, timeout=timeout)
finally:
globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(previous_retries or "4"))
raw = extract_search_results(html)
listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
return MobileDeSearchPage(
@@ -364,7 +376,7 @@ class MobileDeClient:
return ordered_pages
def fetch_detail(self, listing_id: str | int) -> dict:
html = self.fetch_html(self.build_detail_url(listing_id))
html = self.fetch_html(self.build_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS)
return extract_detail_listing(html)
@staticmethod