Openlane/scripts/explore_site.py

"""Open OpenLane in a real browser for manual exploration.

Logs ALL network requests/responses so we can understand auth flow and API structure.
Usage: python scripts/explore_site.py
"""

import json
import sys
from pathlib import Path
from datetime import datetime

from playwright.sync_api import sync_playwright, Page, Request, Response

LOG_DIR = Path("artifacts/explore")
LOG_DIR.mkdir(parents=True, exist_ok=True)
REQUESTS_LOG = LOG_DIR / f"requests_{datetime.now().strftime('%H%M%S')}.jsonl"

captured = []


def on_request(request: Request) -> None:
    entry = {
        "type": "request",
        "ts": datetime.now().isoformat(),
        "method": request.method,
        "url": request.url,
        "resource_type": request.resource_type,
        "headers": dict(request.headers) if request.resource_type in ("xhr", "fetch", "document") else {},
    }
    # Log API and auth requests to console
    url = request.url.lower()
    if any(k in url for k in ("/api/", "/auth", "/token", "/session", "/sign_in", "/login", "/oauth")):
        print(f"\n>>> {request.method} {request.url}")
        if request.post_data:
            # Don't print passwords
            post = request.post_data[:500]
            if "password" in post.lower():
                post = "[CONTAINS PASSWORD - HIDDEN]"
            print(f"    body: {post}")
            entry["post_data"] = post if "password" not in post.lower() else "[HIDDEN]"
    captured.append(entry)


def on_response(response: Response) -> None:
    url = response.url.lower()
    if any(k in url for k in ("/api/", "/auth", "/token", "/session", "/sign_in", "/login", "/oauth")):
        print(f"<<< {response.status} {response.url}")
        entry = {
            "type": "response",
            "ts": datetime.now().isoformat(),
            "status": response.status,
            "url": response.url,
            "headers": dict(response.headers),
        }
        # Try to capture response body for API/auth endpoints
        try:
            body = response.text()
            if len(body) > 2000:
                entry["body_preview"] = body[:2000] + "..."
            else:
                entry["body"] = body
            # Print short preview
            preview = body[:300] if len(body) > 300 else body
            print(f"    body: {preview}")
        except Exception:
            entry["body"] = "[could not read]"
        captured.append(entry)


def save_log():
    with open(REQUESTS_LOG, "w", encoding="utf-8") as f:
        for entry in captured:
            f.write(json.dumps(entry, ensure_ascii=False) + "\n")
    print(f"\n[*] Saved {len(captured)} entries to {REQUESTS_LOG}")


def main():
    print("=" * 60)
    print("OpenLane Site Explorer")
    print("=" * 60)
    print("1. Browser will open at OpenLane sign_in page")
    print("2. Log in manually")
    print("3. Browse around — all API/auth requests are logged")
    print("4. When done, close the browser or press Ctrl+C here")
    print("=" * 60)

    with sync_playwright() as p:
        browser = p.chromium.launch(
            headless=False,
            args=[
                "--disable-blink-features=AutomationControlled",
                "--no-default-browser-check",
            ],
        )
        context = browser.new_context(
            viewport={"width": 1920, "height": 1080},
            locale="en-US",
            timezone_id="America/New_York",
            user_agent=(
                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                "AppleWebKit/537.36 (KHTML, like Gecko) "
                "Chrome/135.0.0.0 Safari/537.36"
            ),
        )

        page = context.new_page()
        page.on("request", on_request)
        page.on("response", on_response)

        print("\n[*] Opening https://app.openlane.com/sign_in ...")
        page.goto("https://app.openlane.com/sign_in", wait_until="domcontentloaded")

        print("\n[*] Browser is open. Log in and explore the site.")
        print("[*] I'm watching all network requests...")
        print("[*] When you're done, just close the browser window.\n")

        try:
            # Wait until the browser is closed by user
            page.wait_for_event("close", timeout=0)
        except KeyboardInterrupt:
            print("\n[*] Interrupted by user")
        except Exception:
            pass

        # Capture cookies and storage state before closing
        try:
            cookies = context.cookies()
            storage = context.storage_state()
            
            cookies_file = LOG_DIR / "cookies.json"
            storage_file = LOG_DIR / "storage_state.json"
            
            with open(cookies_file, "w", encoding="utf-8") as f:
                json.dump(cookies, f, indent=2, ensure_ascii=False)
            with open(storage_file, "w", encoding="utf-8") as f:
                json.dump(storage, f, indent=2, ensure_ascii=False)
            
            print(f"\n[*] Cookies saved to {cookies_file}")
            print(f"[*] Storage state saved to {storage_file}")
            
            # Print cookie names
            print(f"\n[*] Cookies ({len(cookies)}):")
            for c in cookies:
                print(f"    {c['name']} = {c['value'][:30]}... (domain={c['domain']}, httpOnly={c.get('httpOnly', '?')})")
        except Exception as e:
            print(f"[!] Could not save state: {e}")

        browser.close()

    save_log()
    print("\n[*] Done! Check artifacts/explore/ for logs.")


if __name__ == "__main__":
    main()