Fast IAAI implementation

2026-04-24 00:20:42 +03:00
parent 2b0cffc118
commit 3cdb717789
31 changed files with 4880 additions and 0 deletions
--- a/iaai_sync_service/settings.py
+++ b/iaai_sync_service/settings.py
@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+
+def _read_env(name: str, default: str | None = None, *, required: bool = False) -> str:
+    value = os.getenv(name, default)
+    if required and (value is None or not value.strip()):
+        raise RuntimeError(f"Missing required environment variable: {name}")
+    if value is None:
+        return ""
+    return value.strip()
+
+
+def _read_int(name: str, default: int) -> int:
+    raw = _read_env(name, str(default))
+    try:
+        return int(raw)
+    except ValueError as exc:
+        raise RuntimeError(f"Environment variable {name} must be integer, got: {raw}") from exc
+
+
+def _read_bool(name: str, default: bool) -> bool:
+    raw = _read_env(name, "true" if default else "false")
+    normalized = raw.strip().lower()
+    if normalized in {"1", "true", "yes", "on"}:
+        return True
+    if normalized in {"0", "false", "no", "off"}:
+        return False
+    raise RuntimeError(f"Environment variable {name} must be boolean, got: {raw}")
+
+
+def _normalize_database_url(raw_url: str) -> str:
+    normalized = raw_url.strip()
+    if normalized.startswith("jdbc:"):
+        normalized = normalized[len("jdbc:") :]
+    if normalized.startswith("postgres://"):
+        normalized = "postgresql+psycopg://" + normalized[len("postgres://") :]
+    elif normalized.startswith("postgresql://"):
+        normalized = "postgresql+psycopg://" + normalized[len("postgresql://") :]
+    return normalized
+
+
+@dataclass(frozen=True)
+class Settings:
+    database_url: str
+    celery_broker_url: str
+    celery_result_backend: str
+    iaai_base_url: str
+    iaai_listing_start_url: str
+    sync_runtime_config_file: Path
+    iaai_session_cookies: str
+    iaai_storage_state_path: Path
+    iaai_login: str
+    iaai_password: str
+    http_timeout: int
+    http_retries: int
+    http_retry_backoff_ms: int
+    fetch_concurrency: int
+    db_commit_batch_size: int
+    session_keepalive_enabled: bool
+    session_keepalive_interval_minutes: int
+    schema_bootstrap_enabled: bool
+    advisory_lock_key: int
+    error_summary_max_len: int
+
+    @classmethod
+    def from_env(cls) -> "Settings":
+        settings = cls(
+            database_url=_normalize_database_url(_read_env("DATABASE_URL", required=True)),
+            celery_broker_url=_read_env("CELERY_BROKER_URL", required=True),
+            celery_result_backend=_read_env("CELERY_RESULT_BACKEND", "redis://redis:6379/1"),
+            iaai_base_url=_read_env("IAAI_BASE_URL", "https://www.iaai.com").rstrip("/"),
+            iaai_listing_start_url=_read_env("IAAI_LISTING_START_URL", ""),
+            sync_runtime_config_file=Path(_read_env("SYNC_RUNTIME_CONFIG_FILE", "sync_runtime_config.json")),
+            iaai_session_cookies=_read_env("IAAI_SESSION_COOKIES", ""),
+            iaai_storage_state_path=Path(_read_env("IAAI_STORAGE_STATE_PATH", "iaai_storage_state.json")),
+            iaai_login=_read_env("IAAI_LOGIN", ""),
+            iaai_password=_read_env("IAAI_PASSWORD", ""),
+            http_timeout=_read_int("IAAI_HTTP_TIMEOUT", 30),
+            http_retries=_read_int("IAAI_HTTP_RETRIES", 2),
+            http_retry_backoff_ms=_read_int("IAAI_HTTP_RETRY_BACKOFF_MS", 700),
+            fetch_concurrency=_read_int("IAAI_FETCH_CONCURRENCY", 8),
+            db_commit_batch_size=_read_int("IAAI_DB_COMMIT_BATCH_SIZE", 20),
+            session_keepalive_enabled=_read_bool("IAAI_SESSION_KEEPALIVE_ENABLED", True),
+            session_keepalive_interval_minutes=_read_int("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES", 15),
+            schema_bootstrap_enabled=_read_bool("IAAI_SCHEMA_BOOTSTRAP_ENABLED", False),
+            advisory_lock_key=_read_int("SYNC_ADVISORY_LOCK_KEY", 7642200),
+            error_summary_max_len=_read_int("SYNC_ERROR_SUMMARY_MAX_LEN", 2000),
+        )
+
+        if settings.http_timeout <= 0:
+            raise RuntimeError("IAAI_HTTP_TIMEOUT must be greater than 0")
+        if settings.http_retries < 0:
+            raise RuntimeError("IAAI_HTTP_RETRIES cannot be negative")
+        if settings.http_retry_backoff_ms < 0:
+            raise RuntimeError("IAAI_HTTP_RETRY_BACKOFF_MS cannot be negative")
+        if settings.fetch_concurrency <= 0:
+            raise RuntimeError("IAAI_FETCH_CONCURRENCY must be greater than 0")
+        if settings.db_commit_batch_size <= 0:
+            raise RuntimeError("IAAI_DB_COMMIT_BATCH_SIZE must be greater than 0")
+        if settings.session_keepalive_interval_minutes <= 0:
+            raise RuntimeError("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES must be greater than 0")
+        if settings.session_keepalive_interval_minutes > 59:
+            raise RuntimeError("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES must be <= 59")
+        if settings.error_summary_max_len <= 0:
+            raise RuntimeError("SYNC_ERROR_SUMMARY_MAX_LEN must be greater than 0")
+        if settings.iaai_base_url.lower().startswith("http") is False:
+            raise RuntimeError("IAAI_BASE_URL must be absolute URL")
+        if settings.iaai_listing_start_url and not (
+            settings.iaai_listing_start_url.lower().startswith("http")
+            or settings.iaai_listing_start_url.startswith("/")
+        ):
+            raise RuntimeError("IAAI_LISTING_START_URL must be absolute URL or path starting with '/'")
+        return settings
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    return Settings.from_env()