add helper scripts

2026-04-24 20:47:18 +03:00
commit cec1288652
70 changed files with 14249 additions and 0 deletions
--- a/dubizzle_scraper/cli.py
+++ b/dubizzle_scraper/cli.py
@@ -0,0 +1,97 @@
+import argparse
+from pathlib import Path
+
+from .core.config import Settings, parse_listing_segments
+from .core.utils import save_to_json
+from .scraper import DUBIZZLEScraper
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Dubizzle scraper CLI")
+    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
+    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    default_output_dir = Path("artifacts/json")
+
+    subparsers.add_parser("init-db", help="Create DB tables")
+
+    listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
+    listing_parser.add_argument("--make", default=None)
+    listing_parser.add_argument("--model", default=None)
+    listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_listing_links.json"))
+
+    scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
+    scrape_parser.add_argument("vehicle_url")
+    scrape_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_vehicle_detail.json"))
+
+    sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
+    sync_vehicle_parser.add_argument("vehicle_url")
+    sync_vehicle_parser.add_argument("--lane", default="dubizzle")
+    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_vehicle.json"))
+
+    sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
+    sync_listing_parser.add_argument("--make", default=None)
+    sync_listing_parser.add_argument("--model", default=None)
+    sync_listing_parser.add_argument("--lane", default="dubizzle_cars")
+    sync_listing_parser.add_argument("--limit", type=int, default=None)
+    sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
+    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_listing.json"))
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    runtime_settings: Settings | None = None
+    if args.headless is not None or args.debug:
+        runtime_settings = Settings()
+        if args.headless is not None:
+            runtime_settings.headless = args.headless == "true"
+        if args.debug:
+            runtime_settings.log_level = "DEBUG"
+
+    with DUBIZZLEScraper(runtime_settings) as scraper:
+        if args.command == "init-db":
+            data = scraper.init_db()
+            print(f"DB initialized: {data}")
+            return
+        elif args.command == "collect-listing":
+            data = scraper.collect_listing(make=args.make, model=args.model)
+        elif args.command == "scrape-vehicle":
+            data = scraper.scrape_vehicle_detail(args.vehicle_url)
+        elif args.command == "sync-vehicle":
+            data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
+        else:
+            only_new = None if args.only_new is None else args.only_new == "true"
+            segments = parse_listing_segments(scraper.settings.listing.listing_segments_json)
+            use_segmented = (
+                bool(segments)
+                and args.make is None
+                and args.model is None
+                and args.limit is None
+                and scraper.settings.discovery.mode in {"listing", "algolia"}
+            )
+            if use_segmented:
+                data = scraper.sync_listing_segmented(
+                    segments=segments,
+                    lane=args.lane,
+                    only_new=only_new,
+                )
+            else:
+                data = scraper.sync_listing(
+                    make=args.make,
+                    model=args.model,
+                    lane=args.lane,
+                    limit=args.limit,
+                    only_new=only_new,
+                )
+
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+
+
+if __name__ == "__main__":
+    main()