qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix scraper flow

Browse Source
This commit is contained in:
qananasikq
2026-04-22 21:40:40 +03:00
parent 133c125e9c
commit d5d6ba8b7c
33 changed files with 2632 additions and 2811 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
iaai_scraper/cli.py
View File

@@ -38,17 +38,6 @@ def build_parser() -> argparse.ArgumentParser:
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
# Новые команды для ingestion pipeline
subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap")
fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates")
fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process")
enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots")
enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process")
subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)")
return parser
@@ -75,39 +64,6 @@ def main() -> None:
data = scraper.scrape_vehicle_detail(args.vehicle_url)
elif args.command == "sync-vehicle":
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
elif args.command == "sync-listing":
only_new = None if args.only_new is None else args.only_new == "true"
data = scraper.sync_listing(
make=args.make,
model=args.model,
lane=args.lane,
limit=args.limit,
only_new=only_new,
)
elif args.command == "discover-vehicles":
from .discovery_service import DiscoveryService
discovery = DiscoveryService()
count = discovery.discover_new_vehicles()
print(f"Discovered {count} new vehicle candidates")
return
elif args.command == "fetch-pending":
from .fetch_service import FetchService
fetch = FetchService()
count = fetch.process_pending_candidates(limit=args.limit)
print(f"Successfully fetched {count} candidates")
return
elif args.command == "enrich-snapshots":
from .enrichment_service import EnrichmentService
enrichment = EnrichmentService()
count = enrichment.process_unparsed_snapshots(limit=args.limit)
print(f"Successfully enriched {count} snapshots")
return
elif args.command == "run-pipeline":
from .scheduler_service import SchedulerService
scheduler = SchedulerService()
scheduler.run_full_pipeline()
print("Pipeline completed")
return
else:
only_new = None if args.only_new is None else args.only_new == "true"
data = scraper.sync_listing(