qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve mobile.de detail parsing

Browse Source
This commit is contained in:
qananasikq
2026-08-20 13:13:00 +03:00
parent cb49420bb2
commit be19d29a1d
6 changed files with 238 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobilede_scraper/mobile_de/client.py
+1 -1
View File
@@ -36,7 +36,7 @@ MOBILEDE_HTTP_MAX_RETRIES = max(0, int(os.getenv("MOBILEDE_HTTP_MAX_RETRIES", "4
MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_BASE_SECONDS", "1.2")))
MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
MOBILEDE_HTTP_RETRY_STATUSES = {429, 500, 502, 503, 504}
MOBILEDE_DETAIL_TIMEOUT_SECONDS = max(1, int(float(os.getenv("MOBILEDE_DETAIL_TIMEOUT_SECONDS", "15"))))
MOBILEDE_DETAIL_TRANSPORT = os.getenv("MOBILEDE_DETAIL_TRANSPORT", "flight").strip().lower()
mobilede_scraper/mobile_de/scraper.py
+10 -1
View File
@@ -703,7 +703,16 @@ class MobileDeScraper:
images_upserted=images_upserted,
error_summary=str(exc),
)
logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
status_code = getattr(getattr(exc, "response", None), "status_code", None)
if int(status_code or 0) in {401, 403, 429}:
logger.warning(
"mobile.de sync_search blocked: run_id=%s status=%s error=%s",
run_id,
status_code,
exc,
)
else:
logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
raise
def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]: