add openlane scraper

openlane_scraper/openlane/writer.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+class OpenLaneResultWriter:
+    def __init__(self, jsonl_path: str | Path, aggregated_path: str | Path) -> None:
+        self.jsonl_path = Path(jsonl_path)
+        self.aggregated_path = Path(aggregated_path)
+
+    def ensure_parent_dirs(self) -> None:
+        self.jsonl_path.parent.mkdir(parents=True, exist_ok=True)
+        self.aggregated_path.parent.mkdir(parents=True, exist_ok=True)
+
+    def append_page(self, page: int, records: list[dict[str, Any]]) -> int:
+        self.ensure_parent_dirs()
+        written = 0
+        with self.jsonl_path.open("a", encoding="utf-8") as fh:
+            for record in records:
+                payload = {
+                    "page": page,
+                    "record": record,
+                }
+                fh.write(json.dumps(payload, ensure_ascii=False) + "\n")
+                written += 1
+        return written
+
+    def finalize(self, *, max_pages: int, completed_pages: list[int], failed_pages: list[int]) -> dict[str, Any]:
+        self.ensure_parent_dirs()
+        records: list[dict[str, Any]] = []
+        if self.jsonl_path.exists():
+            with self.jsonl_path.open("r", encoding="utf-8") as fh:
+                for line in fh:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    item = json.loads(line)
+                    records.append(item)
+
+        summary = {
+            "max_pages": max_pages,
+            "completed_pages": sorted(completed_pages),
+            "failed_pages": sorted(failed_pages),
+            "total_pages_completed": len(set(completed_pages)),
+            "total_pages_failed": len(set(failed_pages)),
+            "total_records": len(records),
+            "items": records,
+        }
+        self.aggregated_path.write_text(
+            json.dumps(summary, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        return summary