255 lines
10 KiB
Python
255 lines
10 KiB
Python
import json
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from urllib.parse import quote, urlsplit, urlunsplit
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
|
|
TRUE_VALUES = {"1", "true", "yes", "on"}
|
|
|
|
|
|
# Хелперы env.
|
|
|
|
def _env_str(name: str, default: str) -> str:
|
|
value = os.getenv(name)
|
|
return value if value is not None else default
|
|
|
|
|
|
def _env_optional_str(name: str) -> str | None:
|
|
value = os.getenv(name)
|
|
if value is None:
|
|
return None
|
|
value = value.strip()
|
|
return value or None
|
|
|
|
|
|
def _env_path_str(name: str) -> str | None:
|
|
value = _env_optional_str(name)
|
|
if value is None:
|
|
return None
|
|
return str(Path(value).expanduser())
|
|
|
|
|
|
def _env_bool(name: str, default: bool) -> bool:
|
|
fallback = "true" if default else "false"
|
|
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
|
|
|
|
|
|
def _env_int(name: str, default: int) -> int:
|
|
return int(_env_str(name, str(default)).strip())
|
|
|
|
|
|
def _env_float(name: str, default: float) -> float:
|
|
return float(_env_str(name, str(default)).strip())
|
|
|
|
|
|
# Конфиг ссылок.
|
|
|
|
@dataclass(slots=True)
|
|
class ListingConfig:
|
|
cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.mobile.de/Vehiclelisting/Cars")
|
|
filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
|
|
filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
|
|
|
|
@property
|
|
def filtered_search_urls(self) -> list[str]:
|
|
urls: list[str] = []
|
|
if self.filtered_search_url and self.filtered_search_url.strip():
|
|
urls.append(self.filtered_search_url.strip())
|
|
if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip():
|
|
raw = self.filtered_search_urls_raw.strip()
|
|
try:
|
|
parsed = json.loads(raw)
|
|
except (json.JSONDecodeError, ValueError):
|
|
parsed = None
|
|
if isinstance(parsed, list):
|
|
candidates = [str(item or "").strip() for item in parsed]
|
|
else:
|
|
candidates = [part.strip() for part in raw.replace("\n", ",").split(",")]
|
|
for candidate in candidates:
|
|
if candidate and candidate not in urls:
|
|
urls.append(candidate)
|
|
return urls
|
|
|
|
|
|
# Конфиг PostgreSQL.
|
|
|
|
@dataclass(slots=True)
|
|
class DatabaseConfig:
|
|
url: str = _env_str("MOBILEDE_DATABASE_URL", "postgresql+psycopg2://mobilede:MOBILEDE@localhost:5432/MOBILEDE_scraper")
|
|
echo: bool = _env_bool("MOBILEDE_DATABASE_ECHO", False)
|
|
pool_size: int = _env_int("MOBILEDE_DATABASE_POOL_SIZE", 5)
|
|
max_overflow: int = _env_int("MOBILEDE_DATABASE_MAX_OVERFLOW", 10)
|
|
pool_recycle_seconds: int = _env_int("MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
|
|
auto_create_tables: bool = _env_bool("MOBILEDE_DATABASE_AUTO_CREATE_TABLES", False)
|
|
|
|
|
|
# Конфиг Redis.
|
|
|
|
@dataclass(slots=True)
|
|
class RedisConfig:
|
|
url: str = _env_str("MOBILEDE_REDIS_URL", "redis://localhost:6379/0")
|
|
socket_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
|
|
socket_connect_timeout_seconds: float = _env_float("MOBILEDE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
|
|
health_check_interval_seconds: int = _env_int("MOBILEDE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
|
|
|
|
|
|
# Конфиг Celery.
|
|
|
|
@dataclass(slots=True)
|
|
class CeleryConfig:
|
|
broker_url: str = _env_str("CELERY_BROKER_URL", "")
|
|
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
|
|
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
|
|
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
|
|
task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
|
|
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
|
|
worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
|
|
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
|
|
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
|
|
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
|
|
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
|
|
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
|
|
parallel_tabs: int = _env_int("MOBILEDE_PARALLEL_TABS", 8)
|
|
fetch_concurrency: int = _env_int("MOBILEDE_FETCH_CONCURRENCY", _env_int("MOBILEDE_PARALLEL_TABS", 8))
|
|
parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
|
|
block_resources: bool = _env_bool("MOBILEDE_BLOCK_RESOURCES", True)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ScrapingProfileConfig:
|
|
name: str = _env_str("MOBILEDE_SCRAPING_PROFILE", _env_str("MOBILEDE_PROFILE", "stable")).strip().lower()
|
|
http_first: bool = _env_bool("MOBILEDE_HTTP_FIRST", True)
|
|
browser_fallback_enabled: bool = _env_bool("MOBILEDE_BROWSER_FALLBACK_ENABLED", True)
|
|
anonymous_bootstrap_enabled: bool = _env_bool("MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED", True)
|
|
verbose_http_logs: bool = _env_bool("MOBILEDE_VERBOSE_HTTP_LOGS", False)
|
|
verbose_progress_logs: bool = _env_bool("MOBILEDE_VERBOSE_PROGRESS_LOGS", False)
|
|
challenge_refresh_enabled: bool = _env_bool("MOBILEDE_CHALLENGE_REFRESH_ENABLED", True)
|
|
challenge_refresh_attempts: int = _env_int("MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS", 3)
|
|
listing_post_attempts: int = _env_int("MOBILEDE_LISTING_POST_ATTEMPTS", 4)
|
|
request_jitter_max_s: float = _env_float("MOBILEDE_REQUEST_JITTER_MAX_S", 0.15)
|
|
detail_retries: int | None = _env_int("MOBILEDE_DETAIL_RETRIES", -1)
|
|
listing_retries: int | None = _env_int("MOBILEDE_LISTING_RETRIES", -1)
|
|
|
|
def __post_init__(self) -> None:
|
|
if self.name == "fast":
|
|
if "MOBILEDE_BROWSER_FALLBACK_ENABLED" not in os.environ:
|
|
self.browser_fallback_enabled = False
|
|
if "MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
|
|
self.anonymous_bootstrap_enabled = False
|
|
if "MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
|
|
self.challenge_refresh_attempts = 1
|
|
if "MOBILEDE_LISTING_POST_ATTEMPTS" not in os.environ:
|
|
self.listing_post_attempts = 2
|
|
if "MOBILEDE_REQUEST_JITTER_MAX_S" not in os.environ:
|
|
self.request_jitter_max_s = 0.0
|
|
if "MOBILEDE_DETAIL_RETRIES" not in os.environ:
|
|
self.detail_retries = 1
|
|
if "MOBILEDE_LISTING_RETRIES" not in os.environ:
|
|
self.listing_retries = 1
|
|
else:
|
|
if self.detail_retries is not None and self.detail_retries < 0:
|
|
self.detail_retries = None
|
|
if self.listing_retries is not None and self.listing_retries < 0:
|
|
self.listing_retries = None
|
|
|
|
|
|
# Конфиг прокси.
|
|
|
|
@dataclass(slots=True)
|
|
class ProxyConfig:
|
|
server: str | None = _env_optional_str("MOBILEDE_PROXY_SERVER")
|
|
username: str | None = _env_optional_str("MOBILEDE_PROXY_USERNAME")
|
|
password: str | None = _env_optional_str("MOBILEDE_PROXY_PASSWORD")
|
|
|
|
@property
|
|
def enabled(self) -> bool:
|
|
return bool(self.server)
|
|
|
|
def to_playwright_dict(self) -> dict[str, str] | None:
|
|
if not self.server:
|
|
return None
|
|
result: dict[str, str] = {"server": self.server}
|
|
if self.username:
|
|
result["username"] = self.username
|
|
if self.password:
|
|
result["password"] = self.password
|
|
return result
|
|
|
|
def to_requests_proxy_url(self) -> str | None:
|
|
if not self.server:
|
|
return None
|
|
if not self.username:
|
|
return self.server
|
|
|
|
parts = urlsplit(self.server)
|
|
if not parts.scheme or not parts.hostname:
|
|
return self.server
|
|
|
|
username = quote(self.username, safe="")
|
|
password = quote(self.password or "", safe="")
|
|
host = parts.hostname
|
|
if ":" in host and not host.startswith("["):
|
|
host = f"[{host}]"
|
|
if parts.port is not None:
|
|
host = f"{host}:{parts.port}"
|
|
netloc = f"{username}:{password}@{host}"
|
|
return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
|
|
|
|
def to_requests_proxies(self) -> dict[str, str] | None:
|
|
proxy_url = self.to_requests_proxy_url()
|
|
if not proxy_url:
|
|
return None
|
|
return {"http": proxy_url, "https": proxy_url}
|
|
|
|
|
|
# Общие настройки.
|
|
|
|
@dataclass(slots=True)
|
|
class Settings:
|
|
home_url: str = "https://www.mobile.de/"
|
|
default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000)
|
|
network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400)
|
|
fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000)
|
|
fast_path_max_attempts: int = _env_int("MOBILEDE_FAST_PATH_MAX_ATTEMPTS", 1)
|
|
fallback_navigation_timeout_ms: int = _env_int("MOBILEDE_FALLBACK_NAV_TIMEOUT_MS", 15000)
|
|
max_retries: int = _env_int("MOBILEDE_MAX_RETRIES", 3)
|
|
retry_delay_seconds: float = _env_float("MOBILEDE_RETRY_DELAY_SECONDS", 2.5)
|
|
retry_backoff_multiplier: float = _env_float("MOBILEDE_RETRY_BACKOFF_MULTIPLIER", 2.0)
|
|
retry_jitter_seconds: float = _env_float("MOBILEDE_RETRY_JITTER_SECONDS", 0.25)
|
|
headless: bool = _env_bool("MOBILEDE_HEADLESS", True)
|
|
browser_engine: str = _env_str("MOBILEDE_BROWSER_ENGINE", "auto")
|
|
log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
|
|
log_file: str | None = _env_optional_str("MOBILEDE_LOG_FILE")
|
|
enable_trace_id_logs: bool = _env_bool("MOBILEDE_ENABLE_TRACE_ID_LOGS", True)
|
|
sync_only_new: bool = _env_bool("MOBILEDE_SYNC_ONLY_NEW", False)
|
|
raw_output_json: str | None = _env_optional_str("MOBILEDE_RAW_OUTPUT_JSON")
|
|
tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE")
|
|
runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
|
|
scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60)
|
|
listing: ListingConfig = field(default_factory=ListingConfig)
|
|
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
|
redis: RedisConfig = field(default_factory=RedisConfig)
|
|
celery: CeleryConfig = field(default_factory=CeleryConfig)
|
|
proxy: ProxyConfig = field(default_factory=ProxyConfig)
|
|
scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
|
|
|
|
@property
|
|
def parallel_tabs(self) -> int:
|
|
return self.celery.parallel_tabs
|
|
|
|
@property
|
|
def fetch_concurrency(self) -> int:
|
|
return self.celery.fetch_concurrency
|
|
|
|
@property
|
|
def block_resources(self) -> bool:
|
|
return self.celery.block_resources
|
|
|
|
# Глобальные настройки.
|
|
settings = Settings()
|