From 6134b10fd0127fa7923de4c94b889603036baa60 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Fri, 10 Apr 2026 19:59:03 +0300 Subject: [PATCH] add iaai auto-login with session persistence --- .env | 6 ++- .env.example | 4 ++ iaai_scraper/core/config.py | 7 +++ iaai_scraper/scraper.py | 101 ++++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 1 deletion(-) diff --git a/.env b/.env index bf1a6e4..b0d0cb1 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -IAAI_HEADLESS=false +IAAI_HEADLESS=true IAAI_LOG_LEVEL=INFO # IAAI_LOG_FILE=iaai_scraper.log @@ -45,6 +45,10 @@ IAAI_STORAGE_STATE_PATH=storage_state.json IAAI_SESSION_MAX_AGE_DAYS=30 IAAI_SESSION_SAVE_ON_EXIT=true +# IAAI login credentials +IAAI_LOGIN_EMAIL=ofcmkrtzhr@hotmail.com +IAAI_LOGIN_PASSWORD=FMGX3Q58syz8 + # Proxy (HTTP/HTTPS preferred for Playwright) # Chromium does not support SOCKS5 proxy authentication directly. # Use residential or mobile USA proxy. diff --git a/.env.example b/.env.example index eae7bbf..12f9f89 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,10 @@ IAAI_STORAGE_STATE_PATH=storage_state.json IAAI_SESSION_MAX_AGE_DAYS=30 IAAI_SESSION_SAVE_ON_EXIT=true +# IAAI login credentials (for authenticated scraping) +# IAAI_LOGIN_EMAIL=your@email.com +# IAAI_LOGIN_PASSWORD=your_password + # Proxy (HTTP/HTTPS preferred for Playwright) # Chromium does not support SOCKS5 proxy authentication directly. # Use residential or mobile USA proxy. diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py index fc2c2e0..393768e 100644 --- a/iaai_scraper/core/config.py +++ b/iaai_scraper/core/config.py @@ -174,11 +174,18 @@ class SessionConfig: storage_state_path: str | None = _env_optional_str("IAAI_STORAGE_STATE_PATH") max_age_days: int = _env_int("IAAI_SESSION_MAX_AGE_DAYS", 30) save_on_exit: bool = _env_bool("IAAI_SESSION_SAVE_ON_EXIT", True) + login_email: str | None = _env_optional_str("IAAI_LOGIN_EMAIL") + login_password: str | None = _env_optional_str("IAAI_LOGIN_PASSWORD") + login_url: str = _env_str("IAAI_LOGIN_URL", "https://www.iaai.com/Login") @property def enabled(self) -> bool: return bool(self.storage_state_path) + @property + def has_credentials(self) -> bool: + return bool(self.login_email and self.login_password) + # --- Главный объект настроек: собирает все блоки конфигурации --- diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py index 92e3a73..d342d47 100644 --- a/iaai_scraper/scraper.py +++ b/iaai_scraper/scraper.py @@ -148,6 +148,106 @@ class IAAIScraper: except Exception as exc: logger.warning("Failed to save session state: %s", exc) + def _is_logged_in(self, page: Page) -> bool: + """Check if we are logged in by looking for account indicators.""" + try: + # IAAI shows user menu / 'My Account' / 'Sign Out' when logged in + logged_in = page.locator("a[href*='SignOut'], a[href*='signout'], .user-name, .welcome-user, [data-testid='user-menu']").count() > 0 + if not logged_in: + # Also check for Sign In link presence (means NOT logged in) + sign_in_visible = page.locator("a[href*='Login'], a[href*='login']").first.is_visible(timeout=3000) + return not sign_in_visible + return True + except Exception: + return False + + def _perform_login(self, page: Page) -> bool: + """Log in to IAAI using email/password credentials.""" + session_cfg = self.settings.session + if not session_cfg.has_credentials: + logger.warning("No IAAI credentials configured, skipping login") + return False + + logger.info("Logging in to IAAI as %s", session_cfg.login_email) + try: + page.goto(session_cfg.login_url, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=15000) + except PlaywrightTimeoutError: + pass + time.sleep(2) + + # Fill email + email_input = page.locator("input[type='email'], input[name='Email'], input#Email, input[placeholder*='email' i], input[placeholder*='Email']").first + email_input.click() + time.sleep(0.5) + email_input.fill(session_cfg.login_email) + time.sleep(0.5) + + # Fill password + password_input = page.locator("input[type='password'], input[name='Password'], input#Password").first + password_input.click() + time.sleep(0.5) + password_input.fill(session_cfg.login_password) + time.sleep(1) + + # Click login button + login_btn = page.locator("button[type='submit'], input[type='submit'], button:has-text('Sign In'), button:has-text('Log In'), button:has-text('Login')").first + login_btn.click() + + # Wait for navigation after login + try: + page.wait_for_load_state("networkidle", timeout=20000) + except PlaywrightTimeoutError: + pass + time.sleep(3) + + # Check if login succeeded + current_url = page.url + if "login" not in current_url.lower(): + logger.info("Login successful, redirected to %s", current_url) + self._save_storage_state() + return True + + # Still on login page — check for errors + error_text = "" + try: + error_el = page.locator(".validation-summary-errors, .error-message, .alert-danger, [role='alert']").first + error_text = error_el.inner_text(timeout=3000) + except Exception: + pass + logger.error("Login failed. Still on login page. Error: %s", error_text or "unknown") + return False + + except Exception as exc: + logger.error("Login error: %s", exc) + return False + + def _ensure_logged_in(self) -> None: + """Ensure we have an authenticated session. Login if needed.""" + session_cfg = self.settings.session + if not session_cfg.has_credentials: + return + + # Check if saved session is still valid + page = self._get_page() + try: + page.goto(self.settings.home_url, wait_until="domcontentloaded", timeout=60_000) + try: + page.wait_for_load_state("networkidle", timeout=10000) + except PlaywrightTimeoutError: + pass + time.sleep(2) + + if self._is_logged_in(page): + logger.info("Already logged in, session is valid") + return + + logger.info("Not logged in, performing login...") + self._perform_login(page) + finally: + page.close() + def _new_context(self, storage_state: str | None = None) -> BrowserContext: if self.browser is None: self.__enter__() @@ -164,6 +264,7 @@ class IAAIScraper: def _get_unauthenticated_page(self) -> Page: context = self._new_context() + self._ensure_logged_in() return context.new_page() def collect_listing(self, make: str | None = None, model: str | None = None):