import argparse from pathlib import Path from .core.config import Settings from .core.utils import save_to_json from .scraper import IAAIScraper def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="IAAI scraper CLI") parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") subparsers = parser.add_subparsers(dest="command", required=True) default_output_dir = Path("artifacts/json") subparsers.add_parser("init-db", help="Create DB tables") listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page") listing_parser.add_argument("--make", default=None) listing_parser.add_argument("--model", default=None) listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json")) scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page") scrape_parser.add_argument("vehicle_url") scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json")) sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle") sync_vehicle_parser.add_argument("vehicle_url") sync_vehicle_parser.add_argument("--lane", default="iaai") sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json")) sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles") sync_listing_parser.add_argument("--make", default=None) sync_listing_parser.add_argument("--model", default=None) sync_listing_parser.add_argument("--lane", default="iaai_cars") sync_listing_parser.add_argument("--limit", type=int, default=None) sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json")) # Новые команды для ingestion pipeline subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap") fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates") fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process") enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots") enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process") subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)") return parser def main() -> None: parser = build_parser() args = parser.parse_args() runtime_settings: Settings | None = None if args.headless is not None or args.debug: runtime_settings = Settings() if args.headless is not None: runtime_settings.headless = args.headless == "true" if args.debug: runtime_settings.log_level = "DEBUG" with IAAIScraper(runtime_settings) as scraper: if args.command == "init-db": data = scraper.init_db() print(f"DB initialized: {data}") return elif args.command == "collect-listing": data = scraper.collect_listing(make=args.make, model=args.model) elif args.command == "scrape-vehicle": data = scraper.scrape_vehicle_detail(args.vehicle_url) elif args.command == "sync-vehicle": data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) elif args.command == "sync-listing": only_new = None if args.only_new is None else args.only_new == "true" data = scraper.sync_listing( make=args.make, model=args.model, lane=args.lane, limit=args.limit, only_new=only_new, ) elif args.command == "discover-vehicles": from .discovery_service import DiscoveryService discovery = DiscoveryService() count = discovery.discover_new_vehicles() print(f"Discovered {count} new vehicle candidates") return elif args.command == "fetch-pending": from .fetch_service import FetchService fetch = FetchService() count = fetch.process_pending_candidates(limit=args.limit) print(f"Successfully fetched {count} candidates") return elif args.command == "enrich-snapshots": from .enrichment_service import EnrichmentService enrichment = EnrichmentService() count = enrichment.process_unparsed_snapshots(limit=args.limit) print(f"Successfully enriched {count} snapshots") return elif args.command == "run-pipeline": from .scheduler_service import SchedulerService scheduler = SchedulerService() scheduler.run_full_pipeline() print("Pipeline completed") return else: only_new = None if args.only_new is None else args.only_new == "true" data = scraper.sync_listing( make=args.make, model=args.model, lane=args.lane, limit=args.limit, only_new=only_new, ) save_to_json(data, Path(args.output)) print(f"Saved to {Path(args.output).resolve()}") if __name__ == "__main__": main()