Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

iaai_scraper/cli.py

@@ -38,6 +38,17 @@ def build_parser() -> argparse.ArgumentParser:
     sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
     sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
 
+    # Новые команды для ingestion pipeline
+    subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap")
+
+    fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates")
+    fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process")
+
+    enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots")
+    enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process")
+
+    subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)")
+
     return parser
 
 
@@ -64,6 +75,39 @@ def main() -> None:
             data = scraper.scrape_vehicle_detail(args.vehicle_url)
         elif args.command == "sync-vehicle":
             data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
+        elif args.command == "sync-listing":
+            only_new = None if args.only_new is None else args.only_new == "true"
+            data = scraper.sync_listing(
+                make=args.make,
+                model=args.model,
+                lane=args.lane,
+                limit=args.limit,
+                only_new=only_new,
+            )
+        elif args.command == "discover-vehicles":
+            from .discovery_service import DiscoveryService
+            discovery = DiscoveryService()
+            count = discovery.discover_new_vehicles()
+            print(f"Discovered {count} new vehicle candidates")
+            return
+        elif args.command == "fetch-pending":
+            from .fetch_service import FetchService
+            fetch = FetchService()
+            count = fetch.process_pending_candidates(limit=args.limit)
+            print(f"Successfully fetched {count} candidates")
+            return
+        elif args.command == "enrich-snapshots":
+            from .enrichment_service import EnrichmentService
+            enrichment = EnrichmentService()
+            count = enrichment.process_unparsed_snapshots(limit=args.limit)
+            print(f"Successfully enriched {count} snapshots")
+            return
+        elif args.command == "run-pipeline":
+            from .scheduler_service import SchedulerService
+            scheduler = SchedulerService()
+            scheduler.run_full_pipeline()
+            print("Pipeline completed")
+            return
         else:
             only_new = None if args.only_new is None else args.only_new == "true"
             data = scraper.sync_listing(