add mobilede scraper

mobilede_scraper/cli.py

@@ -0,0 +1,127 @@
+import argparse
+from pathlib import Path
+
+from .core.config import Settings
+from .core.utils import save_to_json
+from .mobile_de import MobileDeClient, MobileDeScraper
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="mobile.de scraper CLI")
+    parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
+    parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    default_output_dir = Path("artifacts/json")
+
+    subparsers.add_parser("init-db", help="Create DB tables")
+
+    mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages")
+    mobile_search_parser.add_argument("--page", type=int, default=1)
+    mobile_search_parser.add_argument("--max-pages", type=int, default=1)
+    mobile_search_parser.add_argument("--delay", type=float, default=0.7)
+    mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
+    mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500")
+    mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id")
+    mobile_search_parser.add_argument("--price-min", default=None)
+    mobile_search_parser.add_argument("--price-max", default=None)
+    mobile_search_parser.add_argument("--year-min", default=None)
+    mobile_search_parser.add_argument("--year-max", default=None)
+    mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json"))
+
+    mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page")
+    mobile_detail_parser.add_argument("listing_id")
+    mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json"))
+
+    mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages")
+    mobile_sync_search_parser.add_argument("--page", type=int, default=1)
+    mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1)
+    mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7)
+    mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars")
+    mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
+    mobile_sync_search_parser.add_argument("--make-id", default=None)
+    mobile_sync_search_parser.add_argument("--model-id", default=None)
+    mobile_sync_search_parser.add_argument("--price-min", default=None)
+    mobile_sync_search_parser.add_argument("--price-max", default=None)
+    mobile_sync_search_parser.add_argument("--year-min", default=None)
+    mobile_sync_search_parser.add_argument("--year-max", default=None)
+    mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json"))
+
+    mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page")
+    mobile_sync_detail_parser.add_argument("listing_id")
+    mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars")
+    mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json"))
+
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
+    args = parser.parse_args()
+
+    runtime_settings: Settings | None = None
+    if args.headless is not None or args.debug:
+        runtime_settings = Settings()
+        if args.headless is not None:
+            runtime_settings.headless = args.headless == "true"
+        if args.debug:
+            runtime_settings.log_level = "DEBUG"
+
+    if args.command in {"search", "mobilede-search"}:
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        data = scraper.collect_search(
+            start_page=args.page,
+            max_pages=args.max_pages,
+            search_url=args.search_url,
+            make_id=args.make_id,
+            model_id=args.model_id,
+            price_min=args.price_min,
+            price_max=args.price_max,
+            year_min=args.year_min,
+            year_max=args.year_max,
+        )
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command in {"detail", "mobilede-detail"}:
+        scraper = MobileDeScraper()
+        data = scraper.collect_detail(args.listing_id)
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "sync-search":
+        scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
+        data = scraper.sync_search(
+            start_page=args.page,
+            max_pages=args.max_pages,
+            lane=args.lane,
+            search_url=args.search_url,
+            make_id=args.make_id,
+            model_id=args.model_id,
+            price_min=args.price_min,
+            price_max=args.price_max,
+            year_min=args.year_min,
+            year_max=args.year_max,
+        )
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "sync-detail":
+        scraper = MobileDeScraper()
+        data = scraper.sync_detail(args.listing_id, lane=args.lane)
+        save_to_json(data, Path(args.output))
+        print(f"Saved to {Path(args.output).resolve()}")
+        return
+
+    if args.command == "init-db":
+        scraper = MobileDeScraper()
+        data = scraper.init_db()
+        print(f"DB initialized: {data}")
+        return
+
+
+if __name__ == "__main__":
+    main()