add helper scripts
This commit is contained in:
qananasikq
155
scripts/explore_site.py
Normal file
155
scripts/explore_site.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""Open OpenLane in a real browser for manual exploration.
|
||||
|
||||
Logs ALL network requests/responses so we can understand auth flow and API structure.
|
||||
Usage: python scripts/explore_site.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
from playwright.sync_api import sync_playwright, Page, Request, Response
|
||||
|
||||
LOG_DIR = Path("artifacts/explore")
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
REQUESTS_LOG = LOG_DIR / f"requests_{datetime.now().strftime('%H%M%S')}.jsonl"
|
||||
|
||||
captured = []
|
||||
|
||||
|
||||
def on_request(request: Request) -> None:
|
||||
entry = {
|
||||
"type": "request",
|
||||
"ts": datetime.now().isoformat(),
|
||||
"method": request.method,
|
||||
"url": request.url,
|
||||
"resource_type": request.resource_type,
|
||||
"headers": dict(request.headers) if request.resource_type in ("xhr", "fetch", "document") else {},
|
||||
}
|
||||
# Log API and auth requests to console
|
||||
url = request.url.lower()
|
||||
if any(k in url for k in ("/api/", "/auth", "/token", "/session", "/sign_in", "/login", "/oauth")):
|
||||
print(f"\n>>> {request.method} {request.url}")
|
||||
if request.post_data:
|
||||
# Don't print passwords
|
||||
post = request.post_data[:500]
|
||||
if "password" in post.lower():
|
||||
post = "[CONTAINS PASSWORD - HIDDEN]"
|
||||
print(f" body: {post}")
|
||||
entry["post_data"] = post if "password" not in post.lower() else "[HIDDEN]"
|
||||
captured.append(entry)
|
||||
|
||||
|
||||
def on_response(response: Response) -> None:
|
||||
url = response.url.lower()
|
||||
if any(k in url for k in ("/api/", "/auth", "/token", "/session", "/sign_in", "/login", "/oauth")):
|
||||
print(f"<<< {response.status} {response.url}")
|
||||
entry = {
|
||||
"type": "response",
|
||||
"ts": datetime.now().isoformat(),
|
||||
"status": response.status,
|
||||
"url": response.url,
|
||||
"headers": dict(response.headers),
|
||||
}
|
||||
# Try to capture response body for API/auth endpoints
|
||||
try:
|
||||
body = response.text()
|
||||
if len(body) > 2000:
|
||||
entry["body_preview"] = body[:2000] + "..."
|
||||
else:
|
||||
entry["body"] = body
|
||||
# Print short preview
|
||||
preview = body[:300] if len(body) > 300 else body
|
||||
print(f" body: {preview}")
|
||||
except Exception:
|
||||
entry["body"] = "[could not read]"
|
||||
captured.append(entry)
|
||||
|
||||
|
||||
def save_log():
|
||||
with open(REQUESTS_LOG, "w", encoding="utf-8") as f:
|
||||
for entry in captured:
|
||||
f.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
||||
print(f"\n[*] Saved {len(captured)} entries to {REQUESTS_LOG}")
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("OpenLane Site Explorer")
|
||||
print("=" * 60)
|
||||
print("1. Browser will open at OpenLane sign_in page")
|
||||
print("2. Log in manually")
|
||||
print("3. Browse around — all API/auth requests are logged")
|
||||
print("4. When done, close the browser or press Ctrl+C here")
|
||||
print("=" * 60)
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(
|
||||
headless=False,
|
||||
args=[
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--no-default-browser-check",
|
||||
],
|
||||
)
|
||||
context = browser.new_context(
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
locale="en-US",
|
||||
timezone_id="America/New_York",
|
||||
user_agent=(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/135.0.0.0 Safari/537.36"
|
||||
),
|
||||
)
|
||||
|
||||
page = context.new_page()
|
||||
page.on("request", on_request)
|
||||
page.on("response", on_response)
|
||||
|
||||
print("\n[*] Opening https://app.openlane.com/sign_in ...")
|
||||
page.goto("https://app.openlane.com/sign_in", wait_until="domcontentloaded")
|
||||
|
||||
print("\n[*] Browser is open. Log in and explore the site.")
|
||||
print("[*] I'm watching all network requests...")
|
||||
print("[*] When you're done, just close the browser window.\n")
|
||||
|
||||
try:
|
||||
# Wait until the browser is closed by user
|
||||
page.wait_for_event("close", timeout=0)
|
||||
except KeyboardInterrupt:
|
||||
print("\n[*] Interrupted by user")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Capture cookies and storage state before closing
|
||||
try:
|
||||
cookies = context.cookies()
|
||||
storage = context.storage_state()
|
||||
|
||||
cookies_file = LOG_DIR / "cookies.json"
|
||||
storage_file = LOG_DIR / "storage_state.json"
|
||||
|
||||
with open(cookies_file, "w", encoding="utf-8") as f:
|
||||
json.dump(cookies, f, indent=2, ensure_ascii=False)
|
||||
with open(storage_file, "w", encoding="utf-8") as f:
|
||||
json.dump(storage, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\n[*] Cookies saved to {cookies_file}")
|
||||
print(f"[*] Storage state saved to {storage_file}")
|
||||
|
||||
# Print cookie names
|
||||
print(f"\n[*] Cookies ({len(cookies)}):")
|
||||
for c in cookies:
|
||||
print(f" {c['name']} = {c['value'][:30]}... (domain={c['domain']}, httpOnly={c.get('httpOnly', '?')})")
|
||||
except Exception as e:
|
||||
print(f"[!] Could not save state: {e}")
|
||||
|
||||
browser.close()
|
||||
|
||||
save_log()
|
||||
print("\n[*] Done! Check artifacts/explore/ for logs.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user