update sync
This commit is contained in:
qananasikq
@@ -116,14 +116,34 @@ class HybridSessionAuth:
|
||||
|
||||
while attempt <= retries:
|
||||
try:
|
||||
request_started_at = time.perf_counter()
|
||||
if self._settings.scraping_profile.verbose_http_logs:
|
||||
logger.debug(
|
||||
"HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s",
|
||||
method,
|
||||
url,
|
||||
attempt + 1,
|
||||
retries + 1,
|
||||
timeout,
|
||||
expected_marker,
|
||||
)
|
||||
response = session.request(
|
||||
method=method,
|
||||
url=url,
|
||||
headers=headers,
|
||||
data=data,
|
||||
json=json_body,
|
||||
timeout=timeout,
|
||||
timeout=(min(10, max(1, timeout)), max(1, timeout)),
|
||||
)
|
||||
if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400:
|
||||
logger.debug(
|
||||
"HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s",
|
||||
method,
|
||||
url,
|
||||
response.status_code,
|
||||
time.perf_counter() - request_started_at,
|
||||
expected_marker,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
last_error = exc
|
||||
if attempt >= retries:
|
||||
@@ -159,6 +179,7 @@ class HybridSessionAuth:
|
||||
max_refresh_attempts,
|
||||
)
|
||||
self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
|
||||
logger.info("Retrying HTTP request after Playwright refresh url=%s", url)
|
||||
if refresh_attempts > 1:
|
||||
self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
|
||||
continue
|
||||
@@ -291,7 +312,9 @@ class HybridSessionAuth:
|
||||
|
||||
logger.info("IAAI session challenge detected. Refreshing session via Playwright.")
|
||||
cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
|
||||
logger.info("Playwright refresh returned %d cookies", len(cookies))
|
||||
self._apply_cookies_to_session(target_session, cookies)
|
||||
logger.info("Playwright cookies applied to requests session")
|
||||
|
||||
with self._lock:
|
||||
self._refresh_generation += 1
|
||||
@@ -299,7 +322,7 @@ class HybridSessionAuth:
|
||||
self._anonymous_bootstrap_attempted = True
|
||||
refreshed_generation = self._refresh_generation
|
||||
self._thread_local.session_generation = refreshed_generation
|
||||
self._save_storage_state(target_session)
|
||||
logger.info("Playwright refresh completed generation=%s", refreshed_generation)
|
||||
|
||||
def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
|
||||
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
||||
@@ -316,8 +339,10 @@ class HybridSessionAuth:
|
||||
)
|
||||
page = context.new_page()
|
||||
home_target = self._settings.home_url
|
||||
target = urljoin(self._settings.home_url, "Vehiclelisting/Cars")
|
||||
filtered_urls = self._settings.listing.filtered_search_urls
|
||||
target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars")
|
||||
timeout_ms = max(30_000, self._settings.default_timeout_ms)
|
||||
logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER)
|
||||
page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
|
||||
self._accept_cookie_banner(page)
|
||||
page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
|
||||
@@ -328,13 +353,16 @@ class HybridSessionAuth:
|
||||
timeout_ms=timeout_ms,
|
||||
expected_marker=expected_marker or LISTING_MARKER,
|
||||
)
|
||||
state = context.storage_state()
|
||||
cookies = context.cookies()
|
||||
logger.info("Playwright context returned %d cookies", len(cookies))
|
||||
except PlaywrightTimeoutError as exc:
|
||||
raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
|
||||
finally:
|
||||
browser.close()
|
||||
try:
|
||||
browser.close()
|
||||
except Exception as exc:
|
||||
logger.info("Playwright browser close failed after cookie refresh: %s", exc)
|
||||
|
||||
cookies = state.get("cookies") if isinstance(state, dict) else None
|
||||
if not isinstance(cookies, list) or not cookies:
|
||||
raise RuntimeError("Playwright refresh did not return cookies")
|
||||
return [cookie for cookie in cookies if isinstance(cookie, dict)]
|
||||
@@ -356,7 +384,7 @@ class HybridSessionAuth:
|
||||
def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None:
|
||||
poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
|
||||
navigation_error_count = 0
|
||||
for _ in range(PLAYWRIGHT_REFRESH_POLLS):
|
||||
for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
|
||||
except Exception:
|
||||
@@ -378,8 +406,22 @@ class HybridSessionAuth:
|
||||
body_text=body,
|
||||
expected_marker=expected_marker,
|
||||
):
|
||||
logger.info(
|
||||
"Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s",
|
||||
target,
|
||||
poll_index + 1,
|
||||
PLAYWRIGHT_REFRESH_POLLS,
|
||||
expected_marker,
|
||||
)
|
||||
return
|
||||
try:
|
||||
logger.info(
|
||||
"Playwright session refresh still waiting target=%s poll=%s/%s marker=%s",
|
||||
target,
|
||||
poll_index + 1,
|
||||
PLAYWRIGHT_REFRESH_POLLS,
|
||||
expected_marker,
|
||||
)
|
||||
page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -457,6 +499,14 @@ class IAAIFastClient:
|
||||
)
|
||||
for scope_path in scope_paths:
|
||||
first_page_html = self._fetch_listing_first_page(scope_path)
|
||||
search_scope_path = build_search_scope_path_from_html(first_page_html)
|
||||
if search_scope_path and search_scope_path != scope_path:
|
||||
logger.info(
|
||||
"Resolved listing scope to fast Search URL: scope=%s search_scope=%s",
|
||||
scope_path,
|
||||
search_scope_path,
|
||||
)
|
||||
scope_path = search_scope_path
|
||||
first_page = parse_listing_page(first_page_html)
|
||||
for vehicle in first_page.vehicles:
|
||||
if vehicle.inventory_id in seen_inventory_ids:
|
||||
@@ -599,10 +649,30 @@ def build_brand_scope_paths(brands: set[str]) -> list[str]:
|
||||
def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
|
||||
explicit_scope = listing_start_url.strip()
|
||||
if explicit_scope:
|
||||
return [explicit_scope]
|
||||
if explicit_scope.lower().startswith(("http://", "https://", "/")):
|
||||
return [explicit_scope]
|
||||
return [f"/Search?url={explicit_scope}"]
|
||||
return build_brand_scope_paths(brands)
|
||||
|
||||
|
||||
def build_search_scope_path_from_html(html_text: str) -> str | None:
|
||||
tiny_url = parse_attribute_value(html_text, "data-tinyurl")
|
||||
if tiny_url:
|
||||
return f"/Search?url={tiny_url}"
|
||||
|
||||
data_query_raw = parse_attribute_value(html_text, "data-query")
|
||||
if data_query_raw:
|
||||
try:
|
||||
data_query = json.loads(data_query_raw)
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
data_query = None
|
||||
if isinstance(data_query, dict):
|
||||
url_value = parse_text(data_query.get("Url"))
|
||||
if url_value:
|
||||
return f"/Search?url={url_value}"
|
||||
return None
|
||||
|
||||
|
||||
def parse_listing_page(html_text: str) -> FastListingPage:
|
||||
gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
|
||||
vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
|
||||
@@ -677,6 +747,20 @@ def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def parse_attribute_value(html_text: str, attribute_name: str) -> str | None:
|
||||
escaped_name = re.escape(attribute_name)
|
||||
patterns = (
|
||||
rf"\b{escaped_name}=\"([^\"]*)\"",
|
||||
rf"\b{escaped_name}='([^']*)'",
|
||||
)
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, html_text, flags=re.IGNORECASE)
|
||||
if match is not None:
|
||||
value = html.unescape(match.group(1)).strip()
|
||||
return value or None
|
||||
return None
|
||||
|
||||
|
||||
def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
|
||||
seen_fullres: set[str] = set()
|
||||
images: list[dict[str, str | int]] = []
|
||||
|
||||
Reference in New Issue
Block a user