import argparse from pathlib import Path from .core.config import Settings from .core.utils import save_to_json from .openlane import OpenLaneScrapeRunner from .scraper import OpenLaneScraper def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="OpenLane scraper CLI") parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") subparsers = parser.add_subparsers(dest="command", required=True) subparsers.add_parser("init-db", help="Create DB tables") sync_parser = subparsers.add_parser("sync-listing", help="Sync OpenLane marketplace → DB") sync_parser.add_argument("--lane", default="openlane_marketplace") sync_parser.add_argument("--limit", type=int, default=None) sync_parser.add_argument("--only-new", choices=["true", "false"], default=None) sync_parser.add_argument("--max-pages", type=int, default=None) sync_parser.add_argument("--concurrency", type=int, default=None) sync_parser.add_argument("--output", default=str(Path("artifacts/json/openlane_sync_listing.json"))) scrape_parser = subparsers.add_parser("scrape-openlane", help="Scrape OpenLane marketplace to JSONL files") scrape_parser.add_argument("--max-pages", type=int, default=None) scrape_parser.add_argument("--concurrency", type=int, default=None) scrape_parser.add_argument("--resume", action="store_true") scrape_parser.add_argument("--checkpoint-every-pages", type=int, default=None) subparsers.add_parser("openlane-login", help="Open interactive OpenLane login and persist browser storage state") return parser def main() -> None: parser = build_parser() args = parser.parse_args() runtime_settings: Settings | None = None if args.headless is not None or args.debug: runtime_settings = Settings() if args.headless is not None: runtime_settings.headless = args.headless == "true" if args.debug: runtime_settings.log_level = "DEBUG" if args.command == "openlane-login": runner = OpenLaneScrapeRunner(runtime_settings) storage_state_path = runner.interactive_login() print(f"OpenLane storage state saved to {Path(storage_state_path).resolve()}") return if args.command == "scrape-openlane": runner = OpenLaneScrapeRunner(runtime_settings) result = runner.run( max_pages=args.max_pages, concurrency=args.concurrency, resume=args.resume, checkpoint_every_pages=args.checkpoint_every_pages, ) print(f"OpenLane JSONL saved to {Path(result.jsonl_path).resolve()}") print(f"OpenLane aggregated JSON saved to {Path(result.aggregated_path).resolve()}") print(f"OpenLane checkpoint saved to {Path(result.checkpoint_path).resolve()}") print(f"OpenLane storage state saved to {Path(result.storage_state_path).resolve()}") print( f"OpenLane completed pages={len(result.completed_pages)} failed pages={len(result.failed_pages)} total_records={result.total_records} elapsed={result.elapsed_seconds:.2f}s" ) return with OpenLaneScraper(runtime_settings) as scraper: if args.command == "init-db": data = scraper.init_db() print(f"DB initialized: {data}") return elif args.command == "sync-listing": only_new = None if args.only_new is None else args.only_new == "true" data = scraper.sync_listing( lane=args.lane, limit=args.limit, only_new=only_new, max_pages=args.max_pages, concurrency=args.concurrency, ) save_to_json(data, Path(args.output)) print(f"Saved to {Path(args.output).resolve()}") if __name__ == "__main__": main()