IAAI scraper

iaai_scraper/cli.py

@@ -0,0 +1,104 @@
+import argparse
+from pathlib import Path
+
+from .core.utils import save_to_json
+from .scraper import IAAIScraper
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Public IAAI scraper")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    init_db_parser = subparsers.add_parser("init-db", help="Create local DB tables")
+    init_db_parser.add_argument("--output", default="iaai_db_init.json", help="Path to output JSON")
+
+    listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from Vehiclelisting/Cars")
+    listing_parser.add_argument("--make", default=None, help="Optional make filter")
+    listing_parser.add_argument("--model", default=None, help="Optional model filter")
+    listing_parser.add_argument("--output", default="iaai_listing_links.json", help="Path to output JSON")
+
+    open_parser = subparsers.add_parser(
+        "open-vehicle",
+        help="Open a vehicle page in gentle mode and save only DOM-based hints",
+    )
+    open_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
+    open_parser.add_argument("--output", default="iaai_vehicle_opened.json", help="Path to output JSON")
+
+    scrape_parser = subparsers.add_parser(
+        "scrape-vehicle",
+        help="Open a vehicle page and capture a limited set of likely useful JSON responses",
+    )
+    scrape_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
+    scrape_parser.add_argument("--output", default="iaai_vehicle_detail.json", help="Path to output JSON")
+
+    export_parser = subparsers.add_parser(
+        "export-db-json",
+        help="Scrape a vehicle page and save only the DB-ready car record JSON",
+    )
+    export_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
+    export_parser.add_argument("--output", default="iaai_vehicle_db_record.json", help="Path to output JSON")
+
+    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape one vehicle and upsert it into the DB")
+    sync_vehicle_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
+    sync_vehicle_parser.add_argument("--lane", default="iaai", help="Logical lane name for sync_runs")
+    sync_vehicle_parser.add_argument("--output", default="iaai_sync_vehicle.json", help="Path to output JSON")
+
+    sync_listing_parser = subparsers.add_parser(
+        "sync-listing",
+        help="Collect vehicle URLs from the Cars listing and sync them sequentially",
+    )
+    sync_listing_parser.add_argument("--make", default=None, help="Optional make filter")
+    sync_listing_parser.add_argument("--model", default=None, help="Optional model filter")
+    sync_listing_parser.add_argument("--lane", default="iaai_cars", help="Logical lane name for sync_runs")
+    sync_listing_parser.add_argument("--output", default="iaai_sync_listing.json", help="Path to output JSON")
+
+    daemon_parser = subparsers.add_parser(
+        "run-daemon",
+        help="Run the scraper in a loop, syncing vehicles every N minutes (default 60)",
+    )
+    daemon_parser.add_argument(
+        "--interval", type=int, default=None,
+        help="Override interval in minutes (default from IAAI_SCHEDULER_INTERVAL_MINUTES or 60)",
+    )
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    if args.command == "run-daemon":
+        # daemon: бесконечный цикл
+        from .core.config import Settings
+        runtime_settings = Settings()
+        if args.interval is not None:
+            runtime_settings.scheduler_interval_minutes = args.interval
+        with IAAIScraper(runtime_settings) as scraper:
+            scraper.persistence.create_tables()
+            scraper.run_scheduled()
+        return
+
+    # одноразовый запуск
+    with IAAIScraper() as scraper:
+        if args.command == "init-db":
+            data = scraper.init_db()
+        elif args.command == "collect-listing":
+            data = scraper.collect_listing(make=args.make, model=args.model)
+        elif args.command == "open-vehicle":
+            data = scraper.open_vehicle_page(args.vehicle_url)
+        elif args.command == "scrape-vehicle":
+            data = scraper.scrape_vehicle_detail(args.vehicle_url)
+        elif args.command == "export-db-json":
+            data = scraper.scrape_vehicle_detail(args.vehicle_url).get("db_record", {})
+        elif args.command == "sync-vehicle":
+            data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
+        else:
+            data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane)
+
+        save_to_json(data, Path(args.output))
+        print(f"Saved result to {Path(args.output).resolve()}")
+
+
+if __name__ == "__main__":
+    main()