qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden sync flow, proxy bridge, and data normalization

Browse Source
This commit is contained in:
qananasikq
2026-04-08 18:08:16 +03:00
parent 447a88feed
commit 6821404d79
17 changed files with 683 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
iaai_scraper/core/config.py
View File

@@ -126,6 +126,7 @@ class Settings:
log_file: str | None = os.getenv("IAAI_LOG_FILE") or None
enable_trace_id_logs: bool = os.getenv("IAAI_ENABLE_TRACE_ID_LOGS", "true").strip().lower() in {"1", "true", "yes", "on"}
scheduler_interval_minutes: int = int(os.getenv("IAAI_SCHEDULER_INTERVAL_MINUTES", "60"))
sync_only_new: bool = os.getenv("IAAI_SYNC_ONLY_NEW", "true").strip().lower() in {"1", "true", "yes", "on"}
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
gentle: GentleModeConfig = field(default_factory=GentleModeConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)

4
iaai_scraper/core/utils.py
View File

@@ -7,7 +7,9 @@ from typing import Any, Iterable
def save_to_json(data: Any, filename: str | Path) -> None:
Path(filename).write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
path = Path(filename)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
def short_sleep(a: float = 0.10, b: float = 0.35) -> None: