add antibot guard and retry
This commit is contained in:
10
iaai_scraper/core/exceptions.py
Normal file
10
iaai_scraper/core/exceptions.py
Normal file
@@ -0,0 +1,10 @@
|
||||
class ScraperError(Exception):
|
||||
"""Base scraper exception."""
|
||||
|
||||
|
||||
class AntiBotDetectedError(ScraperError):
|
||||
"""Raised when the target website appears to block automation."""
|
||||
|
||||
|
||||
class SiteStructureChangedError(ScraperError):
|
||||
"""Raised when the page shape changed and required data is missing."""
|
||||
@@ -7,6 +7,8 @@ from typing import Any
|
||||
|
||||
from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
|
||||
|
||||
from .exceptions import AntiBotDetectedError
|
||||
|
||||
logger = logging.getLogger("iaai_scraper.retry")
|
||||
|
||||
# Типы исключений, при которых retry имеет смысл.
|
||||
@@ -16,6 +18,7 @@ RETRYABLE_EXCEPTIONS = (
|
||||
ConnectionError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
AntiBotDetectedError,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -236,12 +236,13 @@ class VehicleParser:
|
||||
|
||||
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
endpoints = [item.get("url", "") for item in responses]
|
||||
dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
|
||||
return {
|
||||
"vin_visible": bool(summary.get("vin")),
|
||||
"images_visible": bool(summary.get("image_urls")),
|
||||
"network_json_count": len(responses),
|
||||
"possible_captcha": False,
|
||||
"possible_antibot": False,
|
||||
"possible_captcha": bool(dom_hints.get("has_captcha_text")),
|
||||
"possible_antibot": bool(dom_hints.get("has_antibot_text")),
|
||||
"observed_endpoints": endpoints[:20],
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user