qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup and fix runtime bugs

Browse Source
This commit is contained in:
qananasikq
2026-04-08 22:54:16 +03:00
parent 90bd4756b3
commit 6e97991c68
21 changed files with 220 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
iaai_scraper/core/__init__.py
View File

@@ -1,4 +1 @@
from .config import * # noqa: F401,F403
from .logs import * # noqa: F401,F403
from .retry import * # noqa: F401,F403
from .utils import * # noqa: F401,F403
__all__: list[str] = []

68
iaai_scraper/core/config.py
View File

@@ -1,6 +1,5 @@
import os
from dataclasses import dataclass, field
from pathlib import Path
from dotenv import load_dotenv
@@ -9,7 +8,7 @@ load_dotenv()
@dataclass(slots=True)
class FingerprintConfig:
# Настройки браузерного отпечатка.
# Браузерный отпечаток.
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
@@ -32,20 +31,16 @@ class FingerprintConfig:
@dataclass(slots=True)
class GentleModeConfig:
# Мягкий режим загрузки страницы и capture.
enabled: bool = os.getenv("IAAI_GENTLE_MODE", "true").strip().lower() in {"1", "true", "yes", "on"}
class CaptureConfig:
# Параметры capture.
capture_same_origin_only: bool = os.getenv("IAAI_CAPTURE_SAME_ORIGIN_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
max_requests: int = int(os.getenv("IAAI_MAX_CAPTURED_REQUESTS", "40"))
max_json_responses: int = int(os.getenv("IAAI_MAX_CAPTURED_JSON_RESPONSES", "30"))
warm_scroll_rounds: int = int(os.getenv("IAAI_WARM_SCROLL_ROUNDS", "1"))
scroll_pause_ms: int = int(os.getenv("IAAI_SCROLL_PAUSE_MS", "900"))
post_open_idle_ms: int = int(os.getenv("IAAI_POST_OPEN_IDLE_MS", "3500"))
@dataclass(slots=True)
class HumanPaceConfig:
# Паузы между действиями для более естественного поведения.
# Паузы между действиями.
enabled: bool = os.getenv("IAAI_HUMAN_PACE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
after_listing_open_min_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MIN_S", "2.5"))
after_listing_open_max_s: float = float(os.getenv("IAAI_AFTER_LISTING_OPEN_MAX_S", "4.5"))
@@ -63,7 +58,7 @@ class HumanPaceConfig:
@dataclass(slots=True)
class ListingConfig:
# Ограничения и режим обхода листинга.
# Лимиты листинга.
cars_url: str = os.getenv("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
max_pages_per_run: int = int(os.getenv("IAAI_MAX_PAGES_PER_RUN", "5"))
max_vehicles_per_run: int = int(os.getenv("IAAI_MAX_VEHICLES_PER_RUN", "100"))
@@ -74,32 +69,43 @@ class ListingConfig:
@dataclass(slots=True)
class DatabaseConfig:
# Параметры подключения к БД.
url: str = os.getenv("IAAI_DATABASE_URL", "sqlite:///iaai_scraper.db")
# Подключение к БД.
url: str = os.getenv("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
echo: bool = os.getenv("IAAI_DATABASE_ECHO", "false").strip().lower() in {"1", "true", "yes", "on"}
pool_size: int = int(os.getenv("IAAI_DATABASE_POOL_SIZE", "5"))
max_overflow: int = int(os.getenv("IAAI_DATABASE_MAX_OVERFLOW", "10"))
@dataclass(slots=True)
class RedisConfig:
# Подключение к Redis.
url: str = os.getenv("IAAI_REDIS_URL", "redis://localhost:6379/0")
@dataclass(slots=True)
class CeleryConfig:
# Настройки Celery.
broker_url: str = os.getenv("CELERY_BROKER_URL", "") or ""
result_backend: str = os.getenv("CELERY_RESULT_BACKEND", "") or ""
task_soft_time_limit: int = int(os.getenv("CELERY_TASK_SOFT_TIME_LIMIT", "600"))
task_time_limit: int = int(os.getenv("CELERY_TASK_TIME_LIMIT", "900"))
worker_concurrency: int = int(os.getenv("CELERY_WORKER_CONCURRENCY", "1"))
beat_sync_interval_minutes: int = int(os.getenv("CELERY_BEAT_SYNC_INTERVAL_MINUTES", "60"))
@dataclass(slots=True)
class ProxyConfig:
"""Proxy settings for Playwright browser.
Supports HTTP, HTTPS and SOCKS5 proxies.
Format: ``protocol://[user:password@]host:port``
Examples:
- ``http://proxy.example.com:8080``
- ``socks5://user:pass@proxy.example.com:1080``
"""
server: str | None = os.getenv("IAAI_PROXY_SERVER") or None
username: str | None = os.getenv("IAAI_PROXY_USERNAME") or None
password: str | None = os.getenv("IAAI_PROXY_PASSWORD") or None
# Настройки прокси.
server: str | None = (os.getenv("IAAI_PROXY_SERVER") or "").strip() or None
username: str | None = (os.getenv("IAAI_PROXY_USERNAME") or "").strip() or None
password: str | None = (os.getenv("IAAI_PROXY_PASSWORD") or "").strip() or None
@property
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
"""Return a dict suitable for Playwright ``proxy=`` kwarg, or *None*."""
# Формат для Playwright.
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
@@ -112,7 +118,7 @@ class ProxyConfig:
@dataclass(slots=True)
class Settings:
# Единая точка всех runtime-настроек приложения.
# Общие настройки.
home_url: str = "https://www.iaai.com/"
default_timeout_ms: int = int(os.getenv("IAAI_TIMEOUT_MS", "45000"))
network_settle_ms: int = int(os.getenv("IAAI_NETWORK_SETTLE_MS", "800"))
@@ -121,19 +127,21 @@ class Settings:
retry_backoff_multiplier: float = float(os.getenv("IAAI_RETRY_BACKOFF_MULTIPLIER", "2.0"))
retry_jitter_seconds: float = float(os.getenv("IAAI_RETRY_JITTER_SECONDS", "0.25"))
headless: bool = os.getenv("IAAI_HEADLESS", "true").strip().lower() in {"1", "true", "yes", "on"}
raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
log_level: str = os.getenv("IAAI_LOG_LEVEL", "INFO")
log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
sync_only_new: bool = os.getenv("IAAI_SYNC_ONLY_NEW", "true").strip().lower() in {"1", "true", "yes", "on"}
raw_output_json: str | None = os.getenv("IAAI_RAW_OUTPUT_JSON") or None
scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
gentle: GentleModeConfig = field(default_factory=GentleModeConfig)
capture: CaptureConfig = field(default_factory=CaptureConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
# Глобальные настройки по умолчанию.
# Настройки по умолчанию.
settings = Settings()

4
iaai_scraper/core/logs.py
View File

@@ -3,24 +3,20 @@ import sys
from contextvars import ContextVar
# Trace id хранится в контексте текущей операции.
TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
class TraceIdFilter(logging.Filter):
# Подмешиваем trace_id в каждую log record.
def filter(self, record: logging.LogRecord) -> bool:
record.trace_id = TRACE_ID.get()
return True
def set_trace_id(trace_id: str) -> None:
# Обновляем trace_id для текущего потока выполнения.
TRACE_ID.set(trace_id)
def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
# Базовая настройка консольного и файлового логирования.
handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
if log_file:
handlers.append(logging.FileHandler(log_file, encoding="utf-8"))

2
iaai_scraper/core/retry.py
View File

@@ -9,7 +9,6 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
logger = logging.getLogger("iaai_scraper.retry")
# Ошибки, которые считаем временными и пригодными для повтора.
RETRYABLE_EXCEPTIONS = (
PlaywrightTimeoutError,
Error,
@@ -25,7 +24,6 @@ def retryable(
backoff_multiplier: float = 2.0,
jitter_seconds: float = 0.0,
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
# Универсальный retry с backoff и небольшим случайным jitter.
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
@wraps(func)
def wrapper(*args: Any, **kwargs: Any) -> Any:

2
iaai_scraper/core/utils.py
View File

@@ -31,7 +31,7 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
return None
# VIN, lot, price regex
# Regex для VIN, lot, price.
VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
LOT_RE = re.compile(r"\b(\d{7,10})\b")
PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")