refactor mobile.de parser, fix country mapping, update README

mobilede_scraper/core/exceptions.py

@@ -1,14 +0,0 @@
-class ScraperError(Exception):
-    """Базовое исключение скрапера."""
-
-
-class AntiBotDetectedError(ScraperError):
-    """Вызывается, когда сайт блокирует автоматизацию."""
-
-
-class SiteStructureChangedError(ScraperError):
-    """Вызывается, когда структура страницы изменилась и данных не хватает."""
-
-
-class ListingResumeError(ScraperError):
-    """Вызывается, когда resume по checkpoint больше недостижим."""

mobilede_scraper/core/retry.py

@@ -1,53 +0,0 @@
-import logging
-import random
-import time
-from collections.abc import Callable
-from functools import wraps
-from typing import Any
-
-from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
-
-from .exceptions import AntiBotDetectedError
-
-logger = logging.getLogger("MOBILEDE_scraper.retry")
-
-# Типы исключений, при которых retry имеет смысл.
-RETRYABLE_EXCEPTIONS = (
-    PlaywrightTimeoutError,
-    Error,
-    ConnectionError,
-    OSError,
-    TimeoutError,
-    AntiBotDetectedError,
-)
-
-
-def retryable(
-    max_attempts: int,
-    delay_seconds: float = 2.5,
-    backoff_multiplier: float = 2.0,
-    jitter_seconds: float = 0.0,
-) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
-    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
-        @wraps(func)
-        def wrapper(*args: Any, **kwargs: Any) -> Any:
-            last_error: Exception | None = None
-            for attempt in range(1, max_attempts + 1):
-                try:
-                    return func(*args, **kwargs)
-                except RETRYABLE_EXCEPTIONS as exc:
-                    last_error = exc
-                    logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
-                    if attempt < max_attempts:
-                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
-                        if jitter_seconds > 0:
-                            sleep_for += random.uniform(0, jitter_seconds)
-                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
-                        time.sleep(sleep_for)
-            if last_error is not None:
-                raise last_error
-            raise RuntimeError("Retry wrapper failed without a captured exception")
-
-        return wrapper
-
-    return decorator

mobilede_scraper/core/utils.py

@@ -1,7 +1,6 @@
-import json
+import json
-import re
 from pathlib import Path
-from typing import Any, Callable, Iterable
+from typing import Any
 
 
 def save_to_json(data: Any, filename: str | Path) -> None:
@@ -10,21 +9,7 @@ def save_to_json(data: Any, filename: str | Path) -> None:
     path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
 
 
-def first_non_empty(values: Iterable[Any]) -> Any | None:
-    for value in values:
-        if value not in (None, "", [], {}, ()):
-            return value
-    return None
-
-
-# Регулярные выражения для VIN, lot и price.
-VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
-LOT_RE = re.compile(r"\b(\d{7,10})\b")
-PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
-
-
 def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
-    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
     found = []
     if _depth >= max_depth:
         return found
@@ -37,36 +22,3 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
         for item in obj:
             found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
     return found
-
-
-def deep_find_all_keys(
-    payloads: list,
-    field_map: dict[str, set[str]],
-    max_depth: int = 64,
-) -> dict[str, list]:
-    """Извлекает все нужные поля за один проход по JSON."""
-    # Готовим обратную карту: нормализованный ключ -> имя поля.
-    reverse: dict[str, str] = {}
-    for field_name, keys in field_map.items():
-        for k in keys:
-            reverse[k.lower()] = field_name
-
-    result: dict[str, list] = {f: [] for f in field_map}
-
-    def _recurse(obj: Any, depth: int) -> None:
-        if depth >= max_depth:
-            return
-        if isinstance(obj, dict):
-            for k, v in obj.items():
-                field = reverse.get(k.lower())
-                if field is not None:
-                    result[field].append(v)
-                _recurse(v, depth + 1)
-        elif isinstance(obj, list):
-            for item in obj:
-                _recurse(item, depth + 1)
-
-    for payload in payloads:
-        _recurse(payload, 0)
-
-    return result