From 05b83b5518b9f55465fa1565fe61db3f0ce03e5e Mon Sep 17 00:00:00 2001 From: qananasikq Date: Fri, 10 Apr 2026 19:55:38 +0300 Subject: [PATCH] add cookie session persistence --- .env | 5 ++++ .env.example | 6 +++++ .gitignore | 1 + iaai_scraper/core/config.py | 14 +++++++++++ iaai_scraper/scraper.py | 46 ++++++++++++++++++++++++++++++++++++- 5 files changed, 71 insertions(+), 1 deletion(-) diff --git a/.env b/.env index b7bcd75..bf1a6e4 100644 --- a/.env +++ b/.env @@ -40,6 +40,11 @@ IAAI_RETRY_DELAY_SECONDS=2.5 IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 IAAI_RETRY_JITTER_SECONDS=0.25 +# Session persistence (cookies) +IAAI_STORAGE_STATE_PATH=storage_state.json +IAAI_SESSION_MAX_AGE_DAYS=30 +IAAI_SESSION_SAVE_ON_EXIT=true + # Proxy (HTTP/HTTPS preferred for Playwright) # Chromium does not support SOCKS5 proxy authentication directly. # Use residential or mobile USA proxy. diff --git a/.env.example b/.env.example index bde417e..eae7bbf 100644 --- a/.env.example +++ b/.env.example @@ -40,6 +40,12 @@ IAAI_RETRY_DELAY_SECONDS=2.5 IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 IAAI_RETRY_JITTER_SECONDS=0.25 +# Session persistence (cookies + localStorage) +# Keeps browser session alive between runs (up to 30 days). +IAAI_STORAGE_STATE_PATH=storage_state.json +IAAI_SESSION_MAX_AGE_DAYS=30 +IAAI_SESSION_SAVE_ON_EXIT=true + # Proxy (HTTP/HTTPS preferred for Playwright) # Chromium does not support SOCKS5 proxy authentication directly. # Use residential or mobile USA proxy. diff --git a/.gitignore b/.gitignore index 640874d..86adc10 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ venv/ *.pyd *.log *.db +storage_state.json .vscode/ *.egg-info/ dist/ diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py index 700a38e..fc2c2e0 100644 --- a/iaai_scraper/core/config.py +++ b/iaai_scraper/core/config.py @@ -167,6 +167,19 @@ class ProxyConfig: return result +# --- Конфиг сессии браузера (куки, storage_state) --- + +@dataclass(slots=True) +class SessionConfig: + storage_state_path: str | None = _env_optional_str("IAAI_STORAGE_STATE_PATH") + max_age_days: int = _env_int("IAAI_SESSION_MAX_AGE_DAYS", 30) + save_on_exit: bool = _env_bool("IAAI_SESSION_SAVE_ON_EXIT", True) + + @property + def enabled(self) -> bool: + return bool(self.storage_state_path) + + # --- Главный объект настроек: собирает все блоки конфигурации --- @dataclass(slots=True) @@ -195,6 +208,7 @@ class Settings: redis: RedisConfig = field(default_factory=RedisConfig) celery: CeleryConfig = field(default_factory=CeleryConfig) proxy: ProxyConfig = field(default_factory=ProxyConfig) + session: SessionConfig = field(default_factory=SessionConfig) # Глобальный синглтон — используется по умолчанию во всех модулях. settings = Settings() diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py index 96fb399..92e3a73 100644 --- a/iaai_scraper/scraper.py +++ b/iaai_scraper/scraper.py @@ -1,8 +1,12 @@ +import json import logging +import os import re import signal import time import uuid +from datetime import datetime, timezone +from pathlib import Path from playwright.sync_api import Error as PlaywrightError from playwright.sync_api import BrowserContext, Page, sync_playwright @@ -86,6 +90,7 @@ class IAAIScraper: def close(self) -> None: if self.context is not None: try: + self._save_storage_state() self.context.close() except PlaywrightError: pass @@ -106,12 +111,51 @@ class IAAIScraper: finally: self.playwright = None + def _load_storage_state(self) -> str | None: + """Load saved browser session (cookies + localStorage) if valid.""" + session_cfg = self.settings.session + if not session_cfg.enabled: + return None + state_path = session_cfg.storage_state_path + if not state_path or not os.path.isfile(state_path): + logger.info("No saved session found at %s", state_path) + return None + try: + mtime = os.path.getmtime(state_path) + age_days = (time.time() - mtime) / 86400 + if age_days > session_cfg.max_age_days: + logger.info("Session expired (%.1f days old, max %d). Starting fresh.", age_days, session_cfg.max_age_days) + os.remove(state_path) + return None + logger.info("Reusing saved session from %s (%.1f days old)", state_path, age_days) + return state_path + except Exception as exc: + logger.warning("Failed to load session state: %s", exc) + return None + + def _save_storage_state(self) -> None: + """Save browser session (cookies + localStorage) to disk.""" + session_cfg = self.settings.session + if not session_cfg.enabled or not session_cfg.save_on_exit: + return + if not self.context: + return + state_path = session_cfg.storage_state_path + try: + Path(state_path).parent.mkdir(parents=True, exist_ok=True) + self.context.storage_state(path=state_path) + logger.info("Session state saved to %s", state_path) + except Exception as exc: + logger.warning("Failed to save session state: %s", exc) + def _new_context(self, storage_state: str | None = None) -> BrowserContext: if self.browser is None: self.__enter__() if self.context: + self._save_storage_state() self.context.close() - self.context = self.browser_factory.create_context(self.browser, storage_state=storage_state) + effective_state = storage_state or self._load_storage_state() + self.context = self.browser_factory.create_context(self.browser, storage_state=effective_state) return self.context def init_db(self):