From 8c441b2aec5b344cf416c6acefd7e47cfbebb0c4 Mon Sep 17 00:00:00 2001 From: duuuuuuuden Date: Fri, 24 Apr 2026 13:43:30 +0300 Subject: [PATCH] Fixed proxies --- iaai_scraper/browser/fast_client.py | 5 +++-- iaai_scraper/core/config.py | 27 +++++++++++++++++++++++++++ tests/test_utils.py | 22 ++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/iaai_scraper/browser/fast_client.py b/iaai_scraper/browser/fast_client.py index 5f656f1..ac872b7 100644 --- a/iaai_scraper/browser/fast_client.py +++ b/iaai_scraper/browser/fast_client.py @@ -191,8 +191,9 @@ class HybridSessionAuth: } ) if self._settings.proxy.enabled: - proxy_url = self._settings.proxy.server - session.proxies.update({"http": proxy_url, "https": proxy_url}) + proxies = self._settings.proxy.to_requests_proxies() + if proxies: + session.proxies.update(proxies) with self._lock: self._bootstrap_session_cookies(session) self._thread_local.session = session diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py index 59b70d6..52ddef2 100644 --- a/iaai_scraper/core/config.py +++ b/iaai_scraper/core/config.py @@ -2,6 +2,7 @@ import json import os from dataclasses import dataclass, field from pathlib import Path +from urllib.parse import quote, urlsplit, urlunsplit from dotenv import load_dotenv @@ -331,6 +332,32 @@ class ProxyConfig: result["password"] = self.password return result + def to_requests_proxy_url(self) -> str | None: + if not self.server: + return None + if not self.username: + return self.server + + parts = urlsplit(self.server) + if not parts.scheme or not parts.hostname: + return self.server + + username = quote(self.username, safe="") + password = quote(self.password or "", safe="") + host = parts.hostname + if ":" in host and not host.startswith("["): + host = f"[{host}]" + if parts.port is not None: + host = f"{host}:{parts.port}" + netloc = f"{username}:{password}@{host}" + return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) + + def to_requests_proxies(self) -> dict[str, str] | None: + proxy_url = self.to_requests_proxy_url() + if not proxy_url: + return None + return {"http": proxy_url, "https": proxy_url} + # Главный объект настроек: собирает все блоки конфигурации diff --git a/tests/test_utils.py b/tests/test_utils.py index cb2d5f5..97f4ff6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,6 +7,7 @@ from iaai_scraper.core.config import ( IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS, + ProxyConfig, build_listing_segments_for_makes, parse_listing_segments, ) @@ -87,5 +88,26 @@ class TestParseListingSegments(unittest.TestCase): self.assertIsNone(lexus[0]["year_min"]) +class TestProxyConfig(unittest.TestCase): + def test_requests_proxy_url_includes_encoded_credentials(self) -> None: + cfg = ProxyConfig( + server="http://144.31.139.6:3128", + username="carsproxy", + password="pass@word:with/slash", + ) + + self.assertEqual( + cfg.to_requests_proxy_url(), + "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128", + ) + self.assertEqual( + cfg.to_requests_proxies(), + { + "http": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128", + "https": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128", + }, + ) + + if __name__ == "__main__": unittest.main()