76 lines
2.5 KiB
Python
76 lines
2.5 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from dubizzle_scraper.scraper import DUBIZZLEScraper
|
|
|
|
|
|
def _load_urls(path: Path) -> list[str]:
|
|
raw = path.read_text(encoding="utf-8")
|
|
payload = json.loads(raw)
|
|
|
|
urls: list[str] = []
|
|
if isinstance(payload, list):
|
|
urls = [str(item).strip() for item in payload if str(item).strip()]
|
|
elif isinstance(payload, dict):
|
|
candidates = payload.get("urls") or payload.get("vehicle_urls") or []
|
|
if isinstance(candidates, list):
|
|
urls = [str(item).strip() for item in candidates if str(item).strip()]
|
|
|
|
# дедуп по порядку
|
|
deduped: list[str] = []
|
|
seen: set[str] = set()
|
|
for url in urls:
|
|
if url in seen:
|
|
continue
|
|
seen.add(url)
|
|
deduped.append(url)
|
|
return deduped
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Sync cars from pre-collected seed URLs")
|
|
parser.add_argument("--input", default="artifacts/json/seed_urls.json", help="JSON file with list of vehicle URLs")
|
|
parser.add_argument("--lane", default="dubizzle_cars", help="Target lane")
|
|
parser.add_argument("--batch-size", type=int, default=100, help="Batch size for sync_batch")
|
|
args = parser.parse_args()
|
|
|
|
input_path = Path(args.input)
|
|
if not input_path.exists():
|
|
raise SystemExit(f"Input file not found: {input_path}")
|
|
|
|
urls = _load_urls(input_path)
|
|
if not urls:
|
|
raise SystemExit("No URLs found in input JSON")
|
|
|
|
total_upserted = 0
|
|
total_failed = 0
|
|
total_images = 0
|
|
failures: list[dict[str, str]] = []
|
|
|
|
with DUBIZZLEScraper() as scraper:
|
|
for i in range(0, len(urls), max(1, args.batch_size)):
|
|
chunk = urls[i:i + max(1, args.batch_size)]
|
|
result = scraper.sync_batch(chunk, lane=args.lane)
|
|
total_upserted += int(result.get("cars_upserted", 0))
|
|
total_failed += int(result.get("cars_failed", 0))
|
|
total_images += int(result.get("images_upserted", 0))
|
|
failures.extend(result.get("failures", []))
|
|
print(f"[{i + 1}-{i + len(chunk)}] upserted={result.get('cars_upserted', 0)} failed={result.get('cars_failed', 0)}")
|
|
|
|
summary = {
|
|
"input": str(input_path),
|
|
"urls_total": len(urls),
|
|
"cars_upserted": total_upserted,
|
|
"cars_failed": total_failed,
|
|
"images_upserted": total_images,
|
|
"failures_count": len(failures),
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|