add antibot guard and retry

This commit is contained in:
qananasikq
2026-04-10 15:10:50 +03:00
parent 7b1501008e
commit acf30fcc20
3 changed files with 16 additions and 2 deletions

View File

@@ -0,0 +1,10 @@
class ScraperError(Exception):
"""Base scraper exception."""
class AntiBotDetectedError(ScraperError):
"""Raised when the target website appears to block automation."""
class SiteStructureChangedError(ScraperError):
"""Raised when the page shape changed and required data is missing."""

View File

@@ -7,6 +7,8 @@ from typing import Any
from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
from .exceptions import AntiBotDetectedError
logger = logging.getLogger("iaai_scraper.retry") logger = logging.getLogger("iaai_scraper.retry")
# Типы исключений, при которых retry имеет смысл. # Типы исключений, при которых retry имеет смысл.
@@ -16,6 +18,7 @@ RETRYABLE_EXCEPTIONS = (
ConnectionError, ConnectionError,
OSError, OSError,
TimeoutError, TimeoutError,
AntiBotDetectedError,
) )

View File

@@ -236,12 +236,13 @@ class VehicleParser:
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]: def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
endpoints = [item.get("url", "") for item in responses] endpoints = [item.get("url", "") for item in responses]
dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
return { return {
"vin_visible": bool(summary.get("vin")), "vin_visible": bool(summary.get("vin")),
"images_visible": bool(summary.get("image_urls")), "images_visible": bool(summary.get("image_urls")),
"network_json_count": len(responses), "network_json_count": len(responses),
"possible_captcha": False, "possible_captcha": bool(dom_hints.get("has_captcha_text")),
"possible_antibot": False, "possible_antibot": bool(dom_hints.get("has_antibot_text")),
"observed_endpoints": endpoints[:20], "observed_endpoints": endpoints[:20],
} }