add openlane scraper

2026-04-21 23:49:56 +03:00
parent 7f72056289
commit 74f12da10a
32 changed files with 4713 additions and 0 deletions
--- a/openlane_scraper/cli.py
+++ b/openlane_scraper/cli.py
@@ -0,0 +1,92 @@
+import argparse
+from pathlib import Path
+
+from .core.config import Settings
+from .core.utils import save_to_json
+from .openlane import OpenLaneScrapeRunner
+from .scraper import OpenLaneScraper
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="OpenLane scraper CLI")
+    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
+    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    subparsers.add_parser("init-db", help="Create DB tables")
+
+    sync_parser = subparsers.add_parser("sync-listing", help="Sync OpenLane marketplace → DB")
+    sync_parser.add_argument("--lane", default="openlane_marketplace")
+    sync_parser.add_argument("--limit", type=int, default=None)
+    sync_parser.add_argument("--only-new", choices=["true", "false"], default=None)
+    sync_parser.add_argument("--max-pages", type=int, default=None)
+    sync_parser.add_argument("--concurrency", type=int, default=None)
+    sync_parser.add_argument("--output", default=str(Path("artifacts/json/openlane_sync_listing.json")))
+
+    scrape_parser = subparsers.add_parser("scrape-openlane", help="Scrape OpenLane marketplace to JSONL files")
+    scrape_parser.add_argument("--max-pages", type=int, default=None)
+    scrape_parser.add_argument("--concurrency", type=int, default=None)
+    scrape_parser.add_argument("--resume", action="store_true")
+    scrape_parser.add_argument("--checkpoint-every-pages", type=int, default=None)
+
+    subparsers.add_parser("openlane-login", help="Open interactive OpenLane login and persist browser storage state")
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    runtime_settings: Settings | None = None
+    if args.headless is not None or args.debug:
+        runtime_settings = Settings()
+        if args.headless is not None:
+            runtime_settings.headless = args.headless == "true"
+        if args.debug:
+            runtime_settings.log_level = "DEBUG"
+
+    if args.command == "openlane-login":
+        runner = OpenLaneScrapeRunner(runtime_settings)
+        storage_state_path = runner.interactive_login()
+        print(f"OpenLane storage state saved to {Path(storage_state_path).resolve()}")
+        return
+
+    if args.command == "scrape-openlane":
+        runner = OpenLaneScrapeRunner(runtime_settings)
+        result = runner.run(
+            max_pages=args.max_pages,
+            concurrency=args.concurrency,
+            resume=args.resume,
+            checkpoint_every_pages=args.checkpoint_every_pages,
+        )
+        print(f"OpenLane JSONL saved to {Path(result.jsonl_path).resolve()}")
+        print(f"OpenLane aggregated JSON saved to {Path(result.aggregated_path).resolve()}")
+        print(f"OpenLane checkpoint saved to {Path(result.checkpoint_path).resolve()}")
+        print(f"OpenLane storage state saved to {Path(result.storage_state_path).resolve()}")
+        print(
+            f"OpenLane completed pages={len(result.completed_pages)} failed pages={len(result.failed_pages)} total_records={result.total_records} elapsed={result.elapsed_seconds:.2f}s"
+        )
+        return
+
+    with OpenLaneScraper(runtime_settings) as scraper:
+        if args.command == "init-db":
+            data = scraper.init_db()
+            print(f"DB initialized: {data}")
+            return
+        elif args.command == "sync-listing":
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+                max_pages=args.max_pages,
+                concurrency=args.concurrency,
+            )
+
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+
+
+if __name__ == "__main__":
+    main()