201 lines
8.2 KiB
Python
201 lines
8.2 KiB
Python
import os
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
|
||
from dotenv import load_dotenv
|
||
|
||
load_dotenv()
|
||
|
||
|
||
TRUE_VALUES = {"1", "true", "yes", "on"}
|
||
|
||
|
||
# Хелперы для чтения env-переменных с приведением типов
|
||
|
||
def _env_str(name: str, default: str) -> str:
|
||
value = os.getenv(name)
|
||
return value if value is not None else default
|
||
|
||
|
||
def _env_optional_str(name: str) -> str | None:
|
||
value = os.getenv(name)
|
||
if value is None:
|
||
return None
|
||
value = value.strip()
|
||
return value or None
|
||
|
||
|
||
def _env_path_str(name: str) -> str | None:
|
||
value = _env_optional_str(name)
|
||
if value is None:
|
||
return None
|
||
return str(Path(value).expanduser())
|
||
|
||
|
||
def _env_bool(name: str, default: bool) -> bool:
|
||
fallback = "true" if default else "false"
|
||
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
|
||
|
||
|
||
def _env_int(name: str, default: int) -> int:
|
||
return int(_env_str(name, str(default)).strip())
|
||
|
||
|
||
def _env_float(name: str, default: float) -> float:
|
||
return float(_env_str(name, str(default)).strip())
|
||
|
||
|
||
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
|
||
|
||
@dataclass(slots=True)
|
||
class FingerprintConfig:
|
||
user_agent: str = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/135.0.0.0 Safari/537.36"
|
||
)
|
||
viewport_presets: tuple[dict[str, int], ...] = (
|
||
{"width": 1920, "height": 1080},
|
||
{"width": 1600, "height": 900},
|
||
{"width": 1536, "height": 864},
|
||
{"width": 1440, "height": 900},
|
||
{"width": 1366, "height": 768},
|
||
)
|
||
timezone_candidates: tuple[str, ...] = (
|
||
"America/New_York",
|
||
"America/Chicago",
|
||
"America/Los_Angeles",
|
||
)
|
||
locale: str = "en-US"
|
||
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
|
||
|
||
|
||
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
|
||
|
||
@dataclass(slots=True)
|
||
class CaptureConfig:
|
||
capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
|
||
max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
|
||
max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
|
||
|
||
|
||
# Конфиг пауз между действиями (имитация человека)
|
||
|
||
@dataclass(slots=True)
|
||
class HumanPaceConfig:
|
||
enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
|
||
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 2.5)
|
||
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 4.5)
|
||
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 2.0)
|
||
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 4.0)
|
||
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.5)
|
||
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 1.5)
|
||
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.3)
|
||
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 1.0)
|
||
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.5)
|
||
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 1.5)
|
||
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 3.0)
|
||
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 6.0)
|
||
|
||
|
||
# Конфиг сбора листинга (URL, лимиты страниц и машин)
|
||
|
||
@dataclass(slots=True)
|
||
class ListingConfig:
|
||
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
|
||
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 5)
|
||
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 100)
|
||
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 200)
|
||
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
|
||
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
|
||
|
||
|
||
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
|
||
|
||
@dataclass(slots=True)
|
||
class DatabaseConfig:
|
||
url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
|
||
echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
|
||
pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
|
||
max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
|
||
pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
|
||
auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
|
||
|
||
|
||
# --- Конфиг Redis (URL для Celery broker) ---
|
||
|
||
@dataclass(slots=True)
|
||
class RedisConfig:
|
||
url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
|
||
|
||
|
||
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
|
||
|
||
@dataclass(slots=True)
|
||
class CeleryConfig:
|
||
broker_url: str = _env_str("CELERY_BROKER_URL", "")
|
||
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
|
||
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 600)
|
||
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 900)
|
||
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 1)
|
||
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 20)
|
||
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
|
||
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
|
||
beat_sync_limit: int = _env_int("CELERY_BEAT_SYNC_LIMIT", 26)
|
||
|
||
|
||
# --- Конфиг прокси (server, username, password) ---
|
||
|
||
@dataclass(slots=True)
|
||
class ProxyConfig:
|
||
server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
|
||
username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
|
||
password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
|
||
|
||
@property
|
||
def enabled(self) -> bool:
|
||
return bool(self.server)
|
||
|
||
def to_playwright_dict(self) -> dict[str, str] | None:
|
||
if not self.server:
|
||
return None
|
||
result: dict[str, str] = {"server": self.server}
|
||
if self.username:
|
||
result["username"] = self.username
|
||
if self.password:
|
||
result["password"] = self.password
|
||
return result
|
||
|
||
|
||
# --- Главный объект настроек: собирает все блоки конфигурации ---
|
||
|
||
@dataclass(slots=True)
|
||
class Settings:
|
||
home_url: str = "https://www.iaai.com/"
|
||
default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
|
||
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 800)
|
||
max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
|
||
retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
|
||
retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
|
||
retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
|
||
headless: bool = _env_bool("IAAI_HEADLESS", True)
|
||
log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
|
||
log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
|
||
enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
|
||
sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", True)
|
||
raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
|
||
tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
|
||
runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
|
||
scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
|
||
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
|
||
capture: CaptureConfig = field(default_factory=CaptureConfig)
|
||
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
|
||
listing: ListingConfig = field(default_factory=ListingConfig)
|
||
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
||
redis: RedisConfig = field(default_factory=RedisConfig)
|
||
celery: CeleryConfig = field(default_factory=CeleryConfig)
|
||
proxy: ProxyConfig = field(default_factory=ProxyConfig)
|
||
|
||
# Глобальный синглтон — используется по умолчанию во всех модулях.
|
||
settings = Settings()
|