add antibot guard and retry
This commit is contained in:
10
iaai_scraper/core/exceptions.py
Normal file
10
iaai_scraper/core/exceptions.py
Normal file
@@ -0,0 +1,10 @@
|
||||
class ScraperError(Exception):
|
||||
"""Base scraper exception."""
|
||||
|
||||
|
||||
class AntiBotDetectedError(ScraperError):
|
||||
"""Raised when the target website appears to block automation."""
|
||||
|
||||
|
||||
class SiteStructureChangedError(ScraperError):
|
||||
"""Raised when the page shape changed and required data is missing."""
|
||||
@@ -7,6 +7,8 @@ from typing import Any
|
||||
|
||||
from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
|
||||
|
||||
from .exceptions import AntiBotDetectedError
|
||||
|
||||
logger = logging.getLogger("iaai_scraper.retry")
|
||||
|
||||
# Типы исключений, при которых retry имеет смысл.
|
||||
@@ -16,6 +18,7 @@ RETRYABLE_EXCEPTIONS = (
|
||||
ConnectionError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
AntiBotDetectedError,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user