a4c93a1df1
improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
219 lines
9.2 KiB
Python
219 lines
9.2 KiB
Python
import os
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
|
||
from dotenv import load_dotenv
|
||
|
||
load_dotenv()
|
||
|
||
|
||
TRUE_VALUES = {"1", "true", "yes", "on"}
|
||
|
||
|
||
# Хелперы для чтения env-переменных с приведением типов
|
||
|
||
def _env_str(name: str, default: str) -> str:
|
||
value = os.getenv(name)
|
||
return value if value is not None else default
|
||
|
||
|
||
def _env_optional_str(name: str) -> str | None:
|
||
value = os.getenv(name)
|
||
if value is None:
|
||
return None
|
||
value = value.strip()
|
||
return value or None
|
||
|
||
|
||
def _env_path_str(name: str) -> str | None:
|
||
value = _env_optional_str(name)
|
||
if value is None:
|
||
return None
|
||
return str(Path(value).expanduser())
|
||
|
||
|
||
def _env_bool(name: str, default: bool) -> bool:
|
||
fallback = "true" if default else "false"
|
||
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
|
||
|
||
|
||
def _env_int(name: str, default: int) -> int:
|
||
return int(_env_str(name, str(default)).strip())
|
||
|
||
|
||
def _env_float(name: str, default: float) -> float:
|
||
return float(_env_str(name, str(default)).strip())
|
||
|
||
|
||
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
|
||
|
||
@dataclass(slots=True)
|
||
class FingerprintConfig:
|
||
user_agent: str = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/135.0.0.0 Safari/537.36"
|
||
)
|
||
viewport_presets: tuple[dict[str, int], ...] = (
|
||
{"width": 1920, "height": 1080},
|
||
{"width": 1600, "height": 900},
|
||
{"width": 1536, "height": 864},
|
||
{"width": 1440, "height": 900},
|
||
{"width": 1366, "height": 768},
|
||
)
|
||
timezone_candidates: tuple[str, ...] = (
|
||
"America/New_York",
|
||
"America/Chicago",
|
||
"America/Los_Angeles",
|
||
)
|
||
locale: str = "en-US"
|
||
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
|
||
|
||
|
||
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
|
||
|
||
@dataclass(slots=True)
|
||
class CaptureConfig:
|
||
capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
|
||
max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
|
||
max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
|
||
|
||
|
||
# Конфиг пауз между действиями (имитация человека)
|
||
|
||
@dataclass(slots=True)
|
||
class HumanPaceConfig:
|
||
enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
|
||
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
|
||
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
|
||
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
|
||
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
|
||
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
|
||
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
|
||
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
|
||
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
|
||
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
|
||
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
|
||
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
|
||
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
|
||
|
||
|
||
# Конфиг сбора листинга (URL, лимиты страниц и машин)
|
||
|
||
@dataclass(slots=True)
|
||
class ListingConfig:
|
||
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
|
||
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
|
||
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
|
||
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
|
||
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
|
||
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
|
||
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
|
||
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
|
||
early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
|
||
|
||
|
||
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
|
||
|
||
@dataclass(slots=True)
|
||
class DatabaseConfig:
|
||
url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
|
||
echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
|
||
pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
|
||
max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
|
||
pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
|
||
auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
|
||
|
||
|
||
# --- Конфиг Redis (URL для Celery broker) ---
|
||
|
||
@dataclass(slots=True)
|
||
class RedisConfig:
|
||
url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
|
||
|
||
|
||
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
|
||
|
||
@dataclass(slots=True)
|
||
class CeleryConfig:
|
||
broker_url: str = _env_str("CELERY_BROKER_URL", "")
|
||
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
|
||
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
|
||
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
|
||
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
|
||
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
|
||
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
|
||
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
|
||
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
|
||
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
|
||
parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
|
||
block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
|
||
|
||
|
||
# --- Конфиг прокси (server, username, password) ---
|
||
|
||
@dataclass(slots=True)
|
||
class ProxyConfig:
|
||
server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
|
||
username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
|
||
password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
|
||
|
||
@property
|
||
def enabled(self) -> bool:
|
||
return bool(self.server)
|
||
|
||
def to_playwright_dict(self) -> dict[str, str] | None:
|
||
if not self.server:
|
||
return None
|
||
result: dict[str, str] = {"server": self.server}
|
||
if self.username:
|
||
result["username"] = self.username
|
||
if self.password:
|
||
result["password"] = self.password
|
||
return result
|
||
|
||
|
||
# --- Главный объект настроек: собирает все блоки конфигурации ---
|
||
|
||
@dataclass(slots=True)
|
||
class Settings:
|
||
home_url: str = "https://www.iaai.com/"
|
||
default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
|
||
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
|
||
fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
|
||
fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
|
||
fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
|
||
max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
|
||
retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
|
||
retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
|
||
retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
|
||
headless: bool = _env_bool("IAAI_HEADLESS", True)
|
||
browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
|
||
log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
|
||
log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
|
||
enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
|
||
sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", True)
|
||
raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
|
||
tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
|
||
runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
|
||
scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
|
||
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
|
||
capture: CaptureConfig = field(default_factory=CaptureConfig)
|
||
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
|
||
listing: ListingConfig = field(default_factory=ListingConfig)
|
||
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
||
redis: RedisConfig = field(default_factory=RedisConfig)
|
||
celery: CeleryConfig = field(default_factory=CeleryConfig)
|
||
proxy: ProxyConfig = field(default_factory=ProxyConfig)
|
||
|
||
@property
|
||
def parallel_tabs(self) -> int:
|
||
return self.celery.parallel_tabs
|
||
|
||
@property
|
||
def block_resources(self) -> bool:
|
||
return self.celery.block_resources
|
||
|
||
# Глобальный синглтон — используется по умолчанию во всех модулях.
|
||
settings = Settings()
|