dubizzle project

scripts/sync_from_seed_urls.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from dubizzle_scraper.scraper import DUBIZZLEScraper
+
+
+def _load_urls(path: Path) -> list[str]:
+    raw = path.read_text(encoding="utf-8")
+    payload = json.loads(raw)
+
+    urls: list[str] = []
+    if isinstance(payload, list):
+        urls = [str(item).strip() for item in payload if str(item).strip()]
+    elif isinstance(payload, dict):
+        candidates = payload.get("urls") or payload.get("vehicle_urls") or []
+        if isinstance(candidates, list):
+            urls = [str(item).strip() for item in candidates if str(item).strip()]
+
+    # дедуп по порядку
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        if url in seen:
+            continue
+        seen.add(url)
+        deduped.append(url)
+    return deduped
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Sync cars from pre-collected seed URLs")
+    parser.add_argument("--input", default="artifacts/json/seed_urls.json", help="JSON file with list of vehicle URLs")
+    parser.add_argument("--lane", default="dubizzle_cars", help="Target lane")
+    parser.add_argument("--batch-size", type=int, default=100, help="Batch size for sync_batch")
+    args = parser.parse_args()
+
+    input_path = Path(args.input)
+    if not input_path.exists():
+        raise SystemExit(f"Input file not found: {input_path}")
+
+    urls = _load_urls(input_path)
+    if not urls:
+        raise SystemExit("No URLs found in input JSON")
+
+    total_upserted = 0
+    total_failed = 0
+    total_images = 0
+    failures: list[dict[str, str]] = []
+
+    with DUBIZZLEScraper() as scraper:
+        for i in range(0, len(urls), max(1, args.batch_size)):
+            chunk = urls[i:i + max(1, args.batch_size)]
+            result = scraper.sync_batch(chunk, lane=args.lane)
+            total_upserted += int(result.get("cars_upserted", 0))
+            total_failed += int(result.get("cars_failed", 0))
+            total_images += int(result.get("images_upserted", 0))
+            failures.extend(result.get("failures", []))
+            print(f"[{i + 1}-{i + len(chunk)}] upserted={result.get('cars_upserted', 0)} failed={result.get('cars_failed', 0)}")
+
+    summary = {
+        "input": str(input_path),
+        "urls_total": len(urls),
+        "cars_upserted": total_upserted,
+        "cars_failed": total_failed,
+        "images_upserted": total_images,
+        "failures_count": len(failures),
+    }
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()