import json import os from dataclasses import dataclass, field from pathlib import Path from urllib.parse import quote, urlsplit, urlunsplit from dotenv import load_dotenv load_dotenv() TRUE_VALUES = {"1", "true", "yes", "on"} # Хелперы env. def _env_str(name: str, default: str) -> str: value = os.getenv(name) return value if value is not None else default def _env_optional_str(name: str) -> str | None: value = os.getenv(name) if value is None: return None value = value.strip() return value or None def _env_path_str(name: str) -> str | None: value = _env_optional_str(name) if value is None: return None return str(Path(value).expanduser()) def _env_bool(name: str, default: bool) -> bool: fallback = "true" if default else "false" return _env_str(name, fallback).strip().lower() in TRUE_VALUES def _env_int(name: str, default: int) -> int: return int(_env_str(name, str(default)).strip()) def _env_float(name: str, default: float) -> float: return float(_env_str(name, str(default)).strip()) # Конфиг ссылок. @dataclass(slots=True) class ListingConfig: cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.mobile.de/Vehiclelisting/Cars") filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL") filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS") @property def filtered_search_urls(self) -> list[str]: urls: list[str] = [] if self.filtered_search_url and self.filtered_search_url.strip(): urls.append(self.filtered_search_url.strip()) if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip(): raw = self.filtered_search_urls_raw.strip() try: parsed = json.loads(raw) except (json.JSONDecodeError, ValueError): parsed = None if isinstance(parsed, list): candidates = [str(item or "").strip() for item in parsed] else: candidates = [part.strip() for part in raw.replace("\n", ",").split(",")] for candidate in candidates: if candidate and candidate not in urls: urls.append(candidate) return urls # Конфиг PostgreSQL. @dataclass(slots=True) class DatabaseConfig: url: str = _env_str("MOBILEDE_DATABASE_URL", "postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper") echo: bool = _env_bool("MOBILEDE_DATABASE_ECHO", False) pool_size: int = _env_int("MOBILEDE_DATABASE_POOL_SIZE", 5) max_overflow: int = _env_int("MOBILEDE_DATABASE_MAX_OVERFLOW", 10) pool_recycle_seconds: int = _env_int("MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS", 1800) auto_create_tables: bool = _env_bool("MOBILEDE_DATABASE_AUTO_CREATE_TABLES", False) # Конфиг Redis. @dataclass(slots=True) class RedisConfig: url: str = _env_str("MOBILEDE_REDIS_URL", "redis://localhost:6379/0") socket_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) socket_connect_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) health_check_interval_seconds: int = _env_int("MOBILEDE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) # Конфиг Celery. @dataclass(slots=True) class CeleryConfig: broker_url: str = _env_str("CELERY_BROKER_URL", "") result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600) worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork") worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) parallel_tabs: int = _env_int("MOBILEDE_PARALLEL_TABS", 8) fetch_concurrency: int = _env_int("MOBILEDE_FETCH_CONCURRENCY", _env_int("MOBILEDE_PARALLEL_TABS", 8)) parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False) block_resources: bool = _env_bool("MOBILEDE_BLOCK_RESOURCES", True) @dataclass(slots=True) class ScrapingProfileConfig: name: str = _env_str("MOBILEDE_SCRAPING_PROFILE", _env_str("MOBILEDE_PROFILE", "stable")).strip().lower() http_first: bool = _env_bool("MOBILEDE_HTTP_FIRST", True) browser_fallback_enabled: bool = _env_bool("MOBILEDE_BROWSER_FALLBACK_ENABLED", True) anonymous_bootstrap_enabled: bool = _env_bool("MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED", True) verbose_http_logs: bool = _env_bool("MOBILEDE_VERBOSE_HTTP_LOGS", False) verbose_progress_logs: bool = _env_bool("MOBILEDE_VERBOSE_PROGRESS_LOGS", False) challenge_refresh_enabled: bool = _env_bool("MOBILEDE_CHALLENGE_REFRESH_ENABLED", True) challenge_refresh_attempts: int = _env_int("MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS", 3) listing_post_attempts: int = _env_int("MOBILEDE_LISTING_POST_ATTEMPTS", 4) request_jitter_max_s: float = _env_float("MOBILEDE_REQUEST_JITTER_MAX_S", 0.15) detail_retries: int | None = _env_int("MOBILEDE_DETAIL_RETRIES", -1) listing_retries: int | None = _env_int("MOBILEDE_LISTING_RETRIES", -1) def __post_init__(self) -> None: if self.name == "fast": if "MOBILEDE_BROWSER_FALLBACK_ENABLED" not in os.environ: self.browser_fallback_enabled = False if "MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ: self.anonymous_bootstrap_enabled = False if "MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ: self.challenge_refresh_attempts = 1 if "MOBILEDE_LISTING_POST_ATTEMPTS" not in os.environ: self.listing_post_attempts = 2 if "MOBILEDE_REQUEST_JITTER_MAX_S" not in os.environ: self.request_jitter_max_s = 0.0 if "MOBILEDE_DETAIL_RETRIES" not in os.environ: self.detail_retries = 1 if "MOBILEDE_LISTING_RETRIES" not in os.environ: self.listing_retries = 1 else: if self.detail_retries is not None and self.detail_retries < 0: self.detail_retries = None if self.listing_retries is not None and self.listing_retries < 0: self.listing_retries = None # Конфиг прокси. @dataclass(slots=True) class ProxyConfig: server: str | None = _env_optional_str("MOBILEDE_PROXY_SERVER") username: str | None = _env_optional_str("MOBILEDE_PROXY_USERNAME") password: str | None = _env_optional_str("MOBILEDE_PROXY_PASSWORD") @property def enabled(self) -> bool: return bool(self.server) def to_playwright_dict(self) -> dict[str, str] | None: if not self.server: return None result: dict[str, str] = {"server": self.server} if self.username: result["username"] = self.username if self.password: result["password"] = self.password return result def to_requests_proxy_url(self) -> str | None: if not self.server: return None if not self.username: return self.server parts = urlsplit(self.server) if not parts.scheme or not parts.hostname: return self.server username = quote(self.username, safe="") password = quote(self.password or "", safe="") host = parts.hostname if ":" in host and not host.startswith("["): host = f"[{host}]" if parts.port is not None: host = f"{host}:{parts.port}" netloc = f"{username}:{password}@{host}" return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment)) def to_requests_proxies(self) -> dict[str, str] | None: proxy_url = self.to_requests_proxy_url() if not proxy_url: return None return {"http": proxy_url, "https": proxy_url} # Общие настройки. @dataclass(slots=True) class Settings: home_url: str = "https://www.mobile.de/" default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000) network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400) fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000) fast_path_max_attempts: int = _env_int("MOBILEDE_FAST_PATH_MAX_ATTEMPTS", 1) fallback_navigation_timeout_ms: int = _env_int("MOBILEDE_FALLBACK_NAV_TIMEOUT_MS", 15000) max_retries: int = _env_int("MOBILEDE_MAX_RETRIES", 3) retry_delay_seconds: float = _env_float("MOBILEDE_RETRY_DELAY_SECONDS", 2.5) retry_backoff_multiplier: float = _env_float("MOBILEDE_RETRY_BACKOFF_MULTIPLIER", 2.0) retry_jitter_seconds: float = _env_float("MOBILEDE_RETRY_JITTER_SECONDS", 0.25) headless: bool = _env_bool("MOBILEDE_HEADLESS", True) browser_engine: str = _env_str("MOBILEDE_BROWSER_ENGINE", "auto") log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO") log_file: str | None = _env_optional_str("MOBILEDE_LOG_FILE") enable_trace_id_logs: bool = _env_bool("MOBILEDE_ENABLE_TRACE_ID_LOGS", True) sync_only_new: bool = _env_bool("MOBILEDE_SYNC_ONLY_NEW", False) raw_output_json: str | None = _env_optional_str("MOBILEDE_RAW_OUTPUT_JSON") tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE") runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE") scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60) listing: ListingConfig = field(default_factory=ListingConfig) database: DatabaseConfig = field(default_factory=DatabaseConfig) redis: RedisConfig = field(default_factory=RedisConfig) celery: CeleryConfig = field(default_factory=CeleryConfig) proxy: ProxyConfig = field(default_factory=ProxyConfig) scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig) @property def parallel_tabs(self) -> int: return self.celery.parallel_tabs @property def fetch_concurrency(self) -> int: return self.celery.fetch_concurrency @property def block_resources(self) -> bool: return self.celery.block_resources # Глобальные настройки. settings = Settings()